PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::INotify;
  38 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  39 use PVE::ProcFSTools;
  40 use PVE::PBSClient;
  41 use PVE::RESTEnvironment qw(log_warn);
  42 use PVE::RPCEnvironment;
  43 use PVE::Storage;
  44 use PVE::SysFSTools;
  45 use PVE::Systemd;
  46 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  47
  48 use PVE::QMPClient;
  49 use PVE::QemuConfig;
  50 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  51 use PVE::QemuServer::Cloudinit;
  52 use PVE::QemuServer::CGroup;
  53 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  54 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  55 use PVE::QemuServer::Machine;
  56 use PVE::QemuServer::Memory;
  57 use PVE::QemuServer::Monitor qw(mon_cmd);
  58 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  59 use PVE::QemuServer::USB qw(parse_usb_device);
  60
  61 my $have_sdn;
  62 eval {
  63     require PVE::Network::SDN::Zones;
  64     $have_sdn = 1;
  65 };
  66
  67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  68 my $OVMF = {
  69     x86_64 => {
  70         '4m-no-smm' => [
  71             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  72             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  73         ],
  74         '4m-no-smm-ms' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  77         ],
  78         '4m' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  81         ],
  82         '4m-ms' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  85         ],
  86         default => [
  87             "$EDK2_FW_BASE/OVMF_CODE.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS.fd",
  89         ],
  90     },
  91     aarch64 => {
  92         default => [
  93             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  94             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  95         ],
  96     },
  97 };
  98
  99 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 100
 101 # Note about locking: we use flock on the config file protect against concurent actions.
 102 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 104 # But you can ignore this kind of lock with the --skiplock flag.
 105
 106 cfs_register_file(
 107     '/qemu-server/',
 108     \&parse_vm_config,
 109     \&write_vm_config
 110 );
 111
 112 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 113     description => "Some command save/restore state from this location.",
 114     type => 'string',
 115     maxLength => 128,
 116     optional => 1,
 117 });
 118
 119 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 120         description => "Specifies the QEMU machine type.",
 121         type => 'string',
 122         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 123         maxLength => 40,
 124         optional => 1,
 125 });
 126
 127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 244             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 245             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 246             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 247             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 248             ." a hardware RNG from the host.",
 249     },
 250     max_bytes => {
 251         type => 'integer',
 252         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 253             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 254             ." `0` to disable limiting (potentially dangerous!).",
 255         optional => 1,
 256
 257         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 258         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 259         # reading from /dev/urandom
 260         default => 1024,
 261     },
 262     period => {
 263         type => 'integer',
 264         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 265             ." the guest to retrieve another 'max_bytes' of entropy.",
 266         optional => 1,
 267         default => 1000,
 268     },
 269 };
 270
 271 my $meta_info_fmt = {
 272     'ctime' => {
 273         type => 'integer',
 274         description => "The guest creation timestamp as UNIX epoch time",
 275         minimum => 0,
 276         optional => 1,
 277     },
 278     'creation-qemu' => {
 279         type => 'string',
 280         description => "The QEMU (machine) version from the time this VM was created.",
 281         pattern => '\d+(\.\d+)+',
 282         optional => 1,
 283     },
 284 };
 285
 286 my $confdesc = {
 287     onboot => {
 288         optional => 1,
 289         type => 'boolean',
 290         description => "Specifies whether a VM will be started during system bootup.",
 291         default => 0,
 292     },
 293     autostart => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Automatic restart after crash (currently ignored).",
 297         default => 0,
 298     },
 299     hotplug => {
 300         optional => 1,
 301         type => 'string', format => 'pve-hotplug-features',
 302         description => "Selectively enable hotplug features. This is a comma separated list of"
 303             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 304             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 305             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 306             ." windows > 7.",
 307         default => 'network,disk,usb',
 308     },
 309     reboot => {
 310         optional => 1,
 311         type => 'boolean',
 312         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 313         default => 1,
 314     },
 315     lock => {
 316         optional => 1,
 317         type => 'string',
 318         description => "Lock/unlock the VM.",
 319         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 320     },
 321     cpulimit => {
 322         optional => 1,
 323         type => 'number',
 324         description => "Limit of CPU usage.",
 325         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 326             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 327         minimum => 0,
 328         maximum => 128,
 329         default => 0,
 330     },
 331     cpuunits => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 335         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 336             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 337             ." weights of all the other running VMs.",
 338         minimum => 1,
 339         maximum => 262144,
 340         default => 'cgroup v1: 1024, cgroup v2: 100',
 341     },
 342     memory => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 346             ." you use the balloon device.",
 347         minimum => 16,
 348         default => 512,
 349     },
 350     balloon => {
 351         optional => 1,
 352         type => 'integer',
 353         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 354         minimum => 0,
 355     },
 356     shares => {
 357         optional => 1,
 358         type => 'integer',
 359         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 360             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 361             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 362         minimum => 0,
 363         maximum => 50000,
 364         default => 1000,
 365     },
 366     keyboard => {
 367         optional => 1,
 368         type => 'string',
 369         description => "Keyboard layout for VNC server. This option is generally not required and"
 370         ." is often better handled from within the guest OS.",
 371         enum => PVE::Tools::kvmkeymaplist(),
 372         default => undef,
 373     },
 374     name => {
 375         optional => 1,
 376         type => 'string', format => 'dns-name',
 377         description => "Set a name for the VM. Only used on the configuration web interface.",
 378     },
 379     scsihw => {
 380         optional => 1,
 381         type => 'string',
 382         description => "SCSI controller model",
 383         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 384         default => 'lsi',
 385     },
 386     description => {
 387         optional => 1,
 388         type => 'string',
 389         description => "Description for the VM. Shown in the web-interface VM's summary."
 390             ." This is saved as comment inside the configuration file.",
 391         maxLength => 1024 * 8,
 392     },
 393     ostype => {
 394         optional => 1,
 395         type => 'string',
 396         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 397         description => "Specify guest operating system.",
 398         verbose_description => <<EODESC,
 399 Specify guest operating system. This is used to enable special
 400 optimization/features for specific operating systems:
 401
 402 [horizontal]
 403 other;; unspecified OS
 404 wxp;; Microsoft Windows XP
 405 w2k;; Microsoft Windows 2000
 406 w2k3;; Microsoft Windows 2003
 407 w2k8;; Microsoft Windows 2008
 408 wvista;; Microsoft Windows Vista
 409 win7;; Microsoft Windows 7
 410 win8;; Microsoft Windows 8/2012/2012r2
 411 win10;; Microsoft Windows 10/2016/2019
 412 win11;; Microsoft Windows 11/2022
 413 l24;; Linux 2.4 Kernel
 414 l26;; Linux 2.6 - 5.X Kernel
 415 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 416 EODESC
 417     },
 418     boot => {
 419         optional => 1,
 420         type => 'string', format => 'pve-qm-boot',
 421         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 422             ." key or 'legacy=' is deprecated.",
 423     },
 424     bootdisk => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-bootdisk',
 427         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 428         pattern => '(ide|sata|scsi|virtio)\d+',
 429     },
 430     smp => {
 431         optional => 1,
 432         type => 'integer',
 433         description => "The number of CPUs. Please use option -sockets instead.",
 434         minimum => 1,
 435         default => 1,
 436     },
 437     sockets => {
 438         optional => 1,
 439         type => 'integer',
 440         description => "The number of CPU sockets.",
 441         minimum => 1,
 442         default => 1,
 443     },
 444     cores => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "The number of cores per socket.",
 448         minimum => 1,
 449         default => 1,
 450     },
 451     numa => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable NUMA.",
 455         default => 0,
 456     },
 457     hugepages => {
 458         optional => 1,
 459         type => 'string',
 460         description => "Enable/disable hugepages memory.",
 461         enum => [qw(any 2 1024)],
 462     },
 463     keephugepages => {
 464         optional => 1,
 465         type => 'boolean',
 466         default => 0,
 467         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 468             ." after VM shutdown and can be used for subsequent starts.",
 469     },
 470     vcpus => {
 471         optional => 1,
 472         type => 'integer',
 473         description => "Number of hotplugged vcpus.",
 474         minimum => 1,
 475         default => 0,
 476     },
 477     acpi => {
 478         optional => 1,
 479         type => 'boolean',
 480         description => "Enable/disable ACPI.",
 481         default => 1,
 482     },
 483     agent => {
 484         optional => 1,
 485         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 486         type => 'string',
 487         format => $agent_fmt,
 488     },
 489     kvm => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Enable/disable KVM hardware virtualization.",
 493         default => 1,
 494     },
 495     tdf => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable time drift fix.",
 499         default => 0,
 500     },
 501     localtime => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 505             ." the `ostype` indicates a Microsoft Windows OS.",
 506     },
 507     freeze => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 511     },
 512     vga => {
 513         optional => 1,
 514         type => 'string', format => $vga_fmt,
 515         description => "Configure the VGA hardware.",
 516         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 517             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 518             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 519             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 520             ." display server. For win* OS you can select how many independent displays you want,"
 521             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 522             ." using a serial device as terminal.",
 523     },
 524     watchdog => {
 525         optional => 1,
 526         type => 'string', format => 'pve-qm-watchdog',
 527         description => "Create a virtual hardware watchdog device.",
 528         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 529             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 530             ." else the watchdog will reset the guest (or execute the respective action specified)",
 531     },
 532     startdate => {
 533         optional => 1,
 534         type => 'string',
 535         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 536         description => "Set the initial date of the real time clock. Valid format for date are:"
 537             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 538         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 539         default => 'now',
 540     },
 541     startup =>  get_standard_option('pve-startup-order'),
 542     template => {
 543         optional => 1,
 544         type => 'boolean',
 545         description => "Enable/disable Template.",
 546         default => 0,
 547     },
 548     args => {
 549         optional => 1,
 550         type => 'string',
 551         description => "Arbitrary arguments passed to kvm.",
 552         verbose_description => <<EODESCR,
 553 Arbitrary arguments passed to kvm, for example:
 554
 555 args: -no-reboot -no-hpet
 556
 557 NOTE: this option is for experts only.
 558 EODESCR
 559     },
 560     tablet => {
 561         optional => 1,
 562         type => 'boolean',
 563         default => 1,
 564         description => "Enable/disable the USB tablet device.",
 565         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 566             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 567             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 568             ." may consider disabling this to save some context switches. This is turned off by"
 569             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 570     },
 571     migrate_speed => {
 572         optional => 1,
 573         type => 'integer',
 574         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 575         minimum => 0,
 576         default => 0,
 577     },
 578     migrate_downtime => {
 579         optional => 1,
 580         type => 'number',
 581         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 582         minimum => 0,
 583         default => 0.1,
 584     },
 585     cdrom => {
 586         optional => 1,
 587         type => 'string', format => 'pve-qm-ide',
 588         typetext => '<volume>',
 589         description => "This is an alias for option -ide2",
 590     },
 591     cpu => {
 592         optional => 1,
 593         description => "Emulated CPU type.",
 594         type => 'string',
 595         format => 'pve-vm-cpu-conf',
 596     },
 597     parent => get_standard_option('pve-snapshot-name', {
 598         optional => 1,
 599         description => "Parent snapshot name. This is used internally, and should not be modified.",
 600     }),
 601     snaptime => {
 602         optional => 1,
 603         description => "Timestamp for snapshots.",
 604         type => 'integer',
 605         minimum => 0,
 606     },
 607     vmstate => {
 608         optional => 1,
 609         type => 'string', format => 'pve-volume-id',
 610         description => "Reference to a volume which stores the VM state. This is used internally"
 611             ." for snapshots.",
 612     },
 613     vmstatestorage => get_standard_option('pve-storage-id', {
 614         description => "Default storage for VM state volumes/files.",
 615         optional => 1,
 616     }),
 617     runningmachine => get_standard_option('pve-qemu-machine', {
 618         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 619             ." for snapshots.",
 620     }),
 621     runningcpu => {
 622         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 623             ." internally for snapshots.",
 624         optional => 1,
 625         type => 'string',
 626         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 627         format_description => 'QEMU -cpu parameter'
 628     },
 629     machine => get_standard_option('pve-qemu-machine'),
 630     arch => {
 631         description => "Virtual processor architecture. Defaults to the host.",
 632         optional => 1,
 633         type => 'string',
 634         enum => [qw(x86_64 aarch64)],
 635     },
 636     smbios1 => {
 637         description => "Specify SMBIOS type 1 fields.",
 638         type => 'string', format => 'pve-qm-smbios1',
 639         maxLength => 512,
 640         optional => 1,
 641     },
 642     protection => {
 643         optional => 1,
 644         type => 'boolean',
 645         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 646             ." remove disk operations.",
 647         default => 0,
 648     },
 649     bios => {
 650         optional => 1,
 651         type => 'string',
 652         enum => [ qw(seabios ovmf) ],
 653         description => "Select BIOS implementation.",
 654         default => 'seabios',
 655     },
 656     vmgenid => {
 657         type => 'string',
 658         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 659         format_description => 'UUID',
 660         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 661             ." to disable explicitly.",
 662         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 663             ." value identifier to the guest OS. This allows to notify the guest operating system"
 664             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 665             ." execution or creation from a template). The guest operating system notices the"
 666             ." change, and is then able to react as appropriate by marking its copies of"
 667             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 668             ."Note that auto-creation only works when done through API/CLI create or update methods"
 669             .", but not when manually editing the config file.",
 670         default => "1 (autogenerated)",
 671         optional => 1,
 672     },
 673     hookscript => {
 674         type => 'string',
 675         format => 'pve-volume-id',
 676         optional => 1,
 677         description => "Script that will be executed during various steps in the vms lifetime.",
 678     },
 679     ivshmem => {
 680         type => 'string',
 681         format => $ivshmem_fmt,
 682         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 683             ." the host.",
 684         optional => 1,
 685     },
 686     audio0 => {
 687         type => 'string',
 688         format => $audio_fmt,
 689         description => "Configure a audio device, useful in combination with QXL/Spice.",
 690         optional => 1
 691     },
 692     spice_enhancements => {
 693         type => 'string',
 694         format => $spice_enhancements_fmt,
 695         description => "Configure additional enhancements for SPICE.",
 696         optional => 1
 697     },
 698     tags => {
 699         type => 'string', format => 'pve-tag-list',
 700         description => 'Tags of the VM. This is only meta information.',
 701         optional => 1,
 702     },
 703     rng0 => {
 704         type => 'string',
 705         format => $rng_fmt,
 706         description => "Configure a VirtIO-based Random Number Generator.",
 707         optional => 1,
 708     },
 709     meta => {
 710         type => 'string',
 711         format => $meta_info_fmt,
 712         description => "Some (read-only) meta-information about this guest.",
 713         optional => 1,
 714     },
 715     affinity => {
 716         type => 'string', format => 'pve-cpuset',
 717         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 718         optional => 1,
 719     },
 720 };
 721
 722 my $cicustom_fmt = {
 723     meta => {
 724         type => 'string',
 725         optional => 1,
 726         description => 'Specify a custom file containing all meta data passed to the VM via"
 727             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731     network => {
 732         type => 'string',
 733         optional => 1,
 734         description => 'Specify a custom file containing all network data passed to the VM via'
 735             .' cloud-init.',
 736         format => 'pve-volume-id',
 737         format_description => 'volume',
 738     },
 739     user => {
 740         type => 'string',
 741         optional => 1,
 742         description => 'Specify a custom file containing all user data passed to the VM via'
 743             .' cloud-init.',
 744         format => 'pve-volume-id',
 745         format_description => 'volume',
 746     },
 747     vendor => {
 748     type => 'string',
 749     optional => 1,
 750     description => 'Specify a custom file containing all vendor data passed to the VM via'
 751      .' cloud-init.',
 752     format => 'pve-volume-id',
 753     format_description => 'volume',
 754     },
 755 };
 756 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 757
 758 my $confdesc_cloudinit = {
 759     citype => {
 760         optional => 1,
 761         type => 'string',
 762         description => 'Specifies the cloud-init configuration format. The default depends on the'
 763             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 764             .' and `configdrive2` for windows.',
 765         enum => ['configdrive2', 'nocloud', 'opennebula'],
 766     },
 767     ciuser => {
 768         optional => 1,
 769         type => 'string',
 770         description => "cloud-init: User name to change ssh keys and password for instead of the"
 771             ." image's configured default user.",
 772     },
 773     cipassword => {
 774         optional => 1,
 775         type => 'string',
 776         description => 'cloud-init: Password to assign the user. Using this is generally not'
 777             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 778             .' support hashed passwords.',
 779     },
 780     cicustom => {
 781         optional => 1,
 782         type => 'string',
 783         description => 'cloud-init: Specify custom files to replace the automatically generated'
 784             .' ones at start.',
 785         format => 'pve-qm-cicustom',
 786     },
 787     searchdomain => {
 788         optional => 1,
 789         type => 'string',
 790         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 791             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 792             .' are set.',
 793     },
 794     nameserver => {
 795         optional => 1,
 796         type => 'string', format => 'address-list',
 797         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 798             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 799             .' are set.',
 800     },
 801     sshkeys => {
 802         optional => 1,
 803         type => 'string',
 804         format => 'urlencoded',
 805         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 806     },
 807 };
 808
 809 # what about other qemu settings ?
 810 #cpu => 'string',
 811 #machine => 'string',
 812 #fda => 'file',
 813 #fdb => 'file',
 814 #mtdblock => 'file',
 815 #sd => 'file',
 816 #pflash => 'file',
 817 #snapshot => 'bool',
 818 #bootp => 'file',
 819 ##tftp => 'dir',
 820 ##smb => 'dir',
 821 #kernel => 'file',
 822 #append => 'string',
 823 #initrd => 'file',
 824 ##soundhw => 'string',
 825
 826 while (my ($k, $v) = each %$confdesc) {
 827     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 828 }
 829
 830 my $MAX_USB_DEVICES = 14;
 831 my $MAX_NETS = 32;
 832 my $MAX_SERIAL_PORTS = 4;
 833 my $MAX_PARALLEL_PORTS = 3;
 834 my $MAX_NUMA = 8;
 835
 836 my $numa_fmt = {
 837     cpus => {
 838         type => "string",
 839         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 840         description => "CPUs accessing this NUMA node.",
 841         format_description => "id[-id];...",
 842     },
 843     memory => {
 844         type => "number",
 845         description => "Amount of memory this NUMA node provides.",
 846         optional => 1,
 847     },
 848     hostnodes => {
 849         type => "string",
 850         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 851         description => "Host NUMA nodes to use.",
 852         format_description => "id[-id];...",
 853         optional => 1,
 854     },
 855     policy => {
 856         type => 'string',
 857         enum => [qw(preferred bind interleave)],
 858         description => "NUMA allocation policy.",
 859         optional => 1,
 860     },
 861 };
 862 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 863 my $numadesc = {
 864     optional => 1,
 865     type => 'string', format => $numa_fmt,
 866     description => "NUMA topology.",
 867 };
 868 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 869
 870 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 871     $confdesc->{"numa$i"} = $numadesc;
 872 }
 873
 874 my $nic_model_list = [
 875     'e1000',
 876     'e1000-82540em',
 877     'e1000-82544gc',
 878     'e1000-82545em',
 879     'e1000e',
 880     'i82551',
 881     'i82557b',
 882     'i82559er',
 883     'ne2k_isa',
 884     'ne2k_pci',
 885     'pcnet',
 886     'rtl8139',
 887     'virtio',
 888     'vmxnet3',
 889 ];
 890 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 891
 892 my $net_fmt_bridge_descr = <<__EOD__;
 893 Bridge to attach the network device to. The Proxmox VE standard bridge
 894 is called 'vmbr0'.
 895
 896 If you do not specify a bridge, we create a kvm user (NATed) network
 897 device, which provides DHCP and DNS services. The following addresses
 898 are used:
 899
 900  10.0.2.2   Gateway
 901  10.0.2.3   DNS Server
 902  10.0.2.4   SMB Server
 903
 904 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 905 __EOD__
 906
 907 my $net_fmt = {
 908     macaddr  => get_standard_option('mac-addr', {
 909         description => "MAC address. That address must be unique withing your network. This is"
 910             ." automatically generated if not specified.",
 911     }),
 912     model => {
 913         type => 'string',
 914         description => "Network Card Model. The 'virtio' model provides the best performance with"
 915             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 916             ." best to use 'e1000'.",
 917         enum => $nic_model_list,
 918         default_key => 1,
 919     },
 920     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 921     bridge => get_standard_option('pve-bridge-id', {
 922         description => $net_fmt_bridge_descr,
 923         optional => 1,
 924     }),
 925     queues => {
 926         type => 'integer',
 927         minimum => 0, maximum => 64,
 928         description => 'Number of packet queues to be used on the device.',
 929         optional => 1,
 930     },
 931     rate => {
 932         type => 'number',
 933         minimum => 0,
 934         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 935         optional => 1,
 936     },
 937     tag => {
 938         type => 'integer',
 939         minimum => 1, maximum => 4094,
 940         description => 'VLAN tag to apply to packets on this interface.',
 941         optional => 1,
 942     },
 943     trunks => {
 944         type => 'string',
 945         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 946         description => 'VLAN trunks to pass through this interface.',
 947         format_description => 'vlanid[;vlanid...]',
 948         optional => 1,
 949     },
 950     firewall => {
 951         type => 'boolean',
 952         description => 'Whether this interface should be protected by the firewall.',
 953         optional => 1,
 954     },
 955     link_down => {
 956         type => 'boolean',
 957         description => 'Whether this interface should be disconnected (like pulling the plug).',
 958         optional => 1,
 959     },
 960     mtu => {
 961         type => 'integer',
 962         minimum => 1, maximum => 65520,
 963         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 964         optional => 1,
 965     },
 966 };
 967
 968 my $netdesc = {
 969     optional => 1,
 970     type => 'string', format => $net_fmt,
 971     description => "Specify network devices.",
 972 };
 973
 974 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 975
 976 my $ipconfig_fmt = {
 977     ip => {
 978         type => 'string',
 979         format => 'pve-ipv4-config',
 980         format_description => 'IPv4Format/CIDR',
 981         description => 'IPv4 address in CIDR format.',
 982         optional => 1,
 983         default => 'dhcp',
 984     },
 985     gw => {
 986         type => 'string',
 987         format => 'ipv4',
 988         format_description => 'GatewayIPv4',
 989         description => 'Default gateway for IPv4 traffic.',
 990         optional => 1,
 991         requires => 'ip',
 992     },
 993     ip6 => {
 994         type => 'string',
 995         format => 'pve-ipv6-config',
 996         format_description => 'IPv6Format/CIDR',
 997         description => 'IPv6 address in CIDR format.',
 998         optional => 1,
 999         default => 'dhcp',
1000     },
1001     gw6 => {
1002         type => 'string',
1003         format => 'ipv6',
1004         format_description => 'GatewayIPv6',
1005         description => 'Default gateway for IPv6 traffic.',
1006         optional => 1,
1007         requires => 'ip6',
1008     },
1009 };
1010 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1011 my $ipconfigdesc = {
1012     optional => 1,
1013     type => 'string', format => 'pve-qm-ipconfig',
1014     description => <<'EODESCR',
1015 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1016
1017 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1018
1019 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1020 gateway should be provided.
1021 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1022 cloud-init 19.4 or newer.
1023
1024 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1025 dhcp on IPv4.
1026 EODESCR
1027 };
1028 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1029
1030 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1031     $confdesc->{"net$i"} = $netdesc;
1032     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1033 }
1034
1035 foreach my $key (keys %$confdesc_cloudinit) {
1036     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1037 }
1038
1039 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1040 sub pve_verify_cpuset {
1041     my ($set_text, $noerr) = @_;
1042
1043     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1044
1045     if ($@) {
1046         return if $noerr;
1047         die "unable to parse cpuset option\n";
1048     }
1049
1050     return PVE::CpuSet->new($members)->short_string();
1051 }
1052
1053 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1054 sub verify_volume_id_or_qm_path {
1055     my ($volid, $noerr) = @_;
1056
1057     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1058
1059     return verify_volume_id_or_absolute_path($volid, $noerr);
1060 }
1061
1062 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1063 sub verify_volume_id_or_absolute_path {
1064     my ($volid, $noerr) = @_;
1065
1066     return $volid if $volid =~ m|^/|;
1067
1068     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1069     if ($@) {
1070         return if $noerr;
1071         die $@;
1072     }
1073     return $volid;
1074 }
1075
1076 my $usb_fmt = {
1077     host => {
1078         default_key => 1,
1079         type => 'string', format => 'pve-qm-usb-device',
1080         format_description => 'HOSTUSBDEVICE|spice',
1081         description => <<EODESCR,
1082 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1083
1084  'bus-port(.port)*' (decimal numbers) or
1085  'vendor_id:product_id' (hexadeciaml numbers) or
1086  'spice'
1087
1088 You can use the 'lsusb -t' command to list existing usb devices.
1089
1090 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1091 machines - use with special care.
1092
1093 The value 'spice' can be used to add a usb redirection devices for spice.
1094 EODESCR
1095     },
1096     usb3 => {
1097         optional => 1,
1098         type => 'boolean',
1099         description => "Specifies whether if given host option is a USB3 device or port."
1100             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1101             ." is irrelevant (all devices are plugged into a xhci controller).",
1102         default => 0,
1103     },
1104 };
1105
1106 my $usbdesc = {
1107     optional => 1,
1108     type => 'string', format => $usb_fmt,
1109     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1110         ." l26 or windows > 7, n can be up to 14).",
1111 };
1112 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1113
1114 my $serialdesc = {
1115         optional => 1,
1116         type => 'string',
1117         pattern => '(/dev/.+|socket)',
1118         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1119         verbose_description =>  <<EODESCR,
1120 Create a serial device inside the VM (n is 0 to 3), and pass through a
1121 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1122 host side (use 'qm terminal' to open a terminal connection).
1123
1124 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1125 use with special care.
1126
1127 CAUTION: Experimental! User reported problems with this option.
1128 EODESCR
1129 };
1130
1131 my $paralleldesc= {
1132         optional => 1,
1133         type => 'string',
1134         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1135         description =>  "Map host parallel devices (n is 0 to 2).",
1136         verbose_description =>  <<EODESCR,
1137 Map host parallel devices (n is 0 to 2).
1138
1139 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1140 machines - use with special care.
1141
1142 CAUTION: Experimental! User reported problems with this option.
1143 EODESCR
1144 };
1145
1146 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1147     $confdesc->{"parallel$i"} = $paralleldesc;
1148 }
1149
1150 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1151     $confdesc->{"serial$i"} = $serialdesc;
1152 }
1153
1154 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1155     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1156 }
1157
1158 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1159     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1160 }
1161
1162 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1163     $confdesc->{"usb$i"} = $usbdesc;
1164 }
1165
1166 my $boot_fmt = {
1167     legacy => {
1168         optional => 1,
1169         default_key => 1,
1170         type => 'string',
1171         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1172                      . " Deprecated, use 'order=' instead.",
1173         pattern => '[acdn]{1,4}',
1174         format_description => "[acdn]{1,4}",
1175
1176         # note: this is also the fallback if boot: is not given at all
1177         default => 'cdn',
1178     },
1179     order => {
1180         optional => 1,
1181         type => 'string',
1182         format => 'pve-qm-bootdev-list',
1183         format_description => "device[;device...]",
1184         description => <<EODESC,
1185 The guest will attempt to boot from devices in the order they appear here.
1186
1187 Disks, optical drives and passed-through storage USB devices will be directly
1188 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1189 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1190
1191 Note that only devices in this list will be marked as bootable and thus loaded
1192 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1193 (e.g. software-raid), you need to specify all of them here.
1194
1195 Overrides the deprecated 'legacy=[acdn]*' value when given.
1196 EODESC
1197     },
1198 };
1199 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1200
1201 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1202 sub verify_bootdev {
1203     my ($dev, $noerr) = @_;
1204
1205     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1206     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1207
1208     my $check = sub {
1209         my ($base) = @_;
1210         return 0 if $dev !~ m/^$base\d+$/;
1211         return 0 if !$confdesc->{$dev};
1212         return 1;
1213     };
1214
1215     return $dev if $check->("net");
1216     return $dev if $check->("usb");
1217     return $dev if $check->("hostpci");
1218
1219     return if $noerr;
1220     die "invalid boot device '$dev'\n";
1221 }
1222
1223 sub print_bootorder {
1224     my ($devs) = @_;
1225     return "" if !@$devs;
1226     my $data = { order => join(';', @$devs) };
1227     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1228 }
1229
1230 my $kvm_api_version = 0;
1231
1232 sub kvm_version {
1233     return $kvm_api_version if $kvm_api_version;
1234
1235     open my $fh, '<', '/dev/kvm' or return;
1236
1237     # 0xae00 => KVM_GET_API_VERSION
1238     $kvm_api_version = ioctl($fh, 0xae00, 0);
1239     close($fh);
1240
1241     return $kvm_api_version;
1242 }
1243
1244 my $kvm_user_version = {};
1245 my $kvm_mtime = {};
1246
1247 sub kvm_user_version {
1248     my ($binary) = @_;
1249
1250     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1251     my $st = stat($binary);
1252
1253     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1254     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1255         $cachedmtime == $st->mtime;
1256
1257     $kvm_user_version->{$binary} = 'unknown';
1258     $kvm_mtime->{$binary} = $st->mtime;
1259
1260     my $code = sub {
1261         my $line = shift;
1262         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1263             $kvm_user_version->{$binary} = $2;
1264         }
1265     };
1266
1267     eval { run_command([$binary, '--version'], outfunc => $code); };
1268     warn $@ if $@;
1269
1270     return $kvm_user_version->{$binary};
1271
1272 }
1273 my sub extract_version {
1274     my ($machine_type, $version) = @_;
1275     $version = kvm_user_version() if !defined($version);
1276     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1277 }
1278
1279 sub kernel_has_vhost_net {
1280     return -c '/dev/vhost-net';
1281 }
1282
1283 sub option_exists {
1284     my $key = shift;
1285     return defined($confdesc->{$key});
1286 }
1287
1288 my $cdrom_path;
1289 sub get_cdrom_path {
1290
1291     return $cdrom_path if defined($cdrom_path);
1292
1293     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1294
1295     if (!defined($cdrom_path)) {
1296         log_warn("no physical CD-ROM available, ignoring");
1297         $cdrom_path = '';
1298     }
1299
1300     return $cdrom_path;
1301 }
1302
1303 sub get_iso_path {
1304     my ($storecfg, $vmid, $cdrom) = @_;
1305
1306     if ($cdrom eq 'cdrom') {
1307         return get_cdrom_path();
1308     } elsif ($cdrom eq 'none') {
1309         return '';
1310     } elsif ($cdrom =~ m|^/|) {
1311         return $cdrom;
1312     } else {
1313         return PVE::Storage::path($storecfg, $cdrom);
1314     }
1315 }
1316
1317 # try to convert old style file names to volume IDs
1318 sub filename_to_volume_id {
1319     my ($vmid, $file, $media) = @_;
1320
1321      if (!($file eq 'none' || $file eq 'cdrom' ||
1322           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1323
1324         return if $file =~ m|/|;
1325
1326         if ($media && $media eq 'cdrom') {
1327             $file = "local:iso/$file";
1328         } else {
1329             $file = "local:$vmid/$file";
1330         }
1331     }
1332
1333     return $file;
1334 }
1335
1336 sub verify_media_type {
1337     my ($opt, $vtype, $media) = @_;
1338
1339     return if !$media;
1340
1341     my $etype;
1342     if ($media eq 'disk') {
1343         $etype = 'images';
1344     } elsif ($media eq 'cdrom') {
1345         $etype = 'iso';
1346     } else {
1347         die "internal error";
1348     }
1349
1350     return if ($vtype eq $etype);
1351
1352     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1353 }
1354
1355 sub cleanup_drive_path {
1356     my ($opt, $storecfg, $drive) = @_;
1357
1358     # try to convert filesystem paths to volume IDs
1359
1360     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1361         ($drive->{file} !~ m|^/dev/.+|) &&
1362         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1363         ($drive->{file} !~ m/^\d+$/)) {
1364         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1365         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1366             if !$vtype;
1367         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1368         verify_media_type($opt, $vtype, $drive->{media});
1369         $drive->{file} = $volid;
1370     }
1371
1372     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1373 }
1374
1375 sub parse_hotplug_features {
1376     my ($data) = @_;
1377
1378     my $res = {};
1379
1380     return $res if $data eq '0';
1381
1382     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1383
1384     foreach my $feature (PVE::Tools::split_list($data)) {
1385         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1386             $res->{$1} = 1;
1387         } else {
1388             die "invalid hotplug feature '$feature'\n";
1389         }
1390     }
1391     return $res;
1392 }
1393
1394 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1395 sub pve_verify_hotplug_features {
1396     my ($value, $noerr) = @_;
1397
1398     return $value if parse_hotplug_features($value);
1399
1400     return if $noerr;
1401
1402     die "unable to parse hotplug option\n";
1403 }
1404
1405 sub scsi_inquiry {
1406     my($fh, $noerr) = @_;
1407
1408     my $SG_IO = 0x2285;
1409     my $SG_GET_VERSION_NUM = 0x2282;
1410
1411     my $versionbuf = "\x00" x 8;
1412     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1413     if (!$ret) {
1414         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1415         return;
1416     }
1417     my $version = unpack("I", $versionbuf);
1418     if ($version < 30000) {
1419         die "scsi generic interface too old\n"  if !$noerr;
1420         return;
1421     }
1422
1423     my $buf = "\x00" x 36;
1424     my $sensebuf = "\x00" x 8;
1425     my $cmd = pack("C x3 C x1", 0x12, 36);
1426
1427     # see /usr/include/scsi/sg.h
1428     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1429
1430     my $packet = pack(
1431         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1432     );
1433
1434     $ret = ioctl($fh, $SG_IO, $packet);
1435     if (!$ret) {
1436         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1437         return;
1438     }
1439
1440     my @res = unpack($sg_io_hdr_t, $packet);
1441     if ($res[17] || $res[18]) {
1442         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1443         return;
1444     }
1445
1446     my $res = {};
1447     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1448
1449     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1450     $res->{type} &= 0x1F;
1451
1452     return $res;
1453 }
1454
1455 sub path_is_scsi {
1456     my ($path) = @_;
1457
1458     my $fh = IO::File->new("+<$path") || return;
1459     my $res = scsi_inquiry($fh, 1);
1460     close($fh);
1461
1462     return $res;
1463 }
1464
1465 sub print_tabletdevice_full {
1466     my ($conf, $arch) = @_;
1467
1468     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1469
1470     # we use uhci for old VMs because tablet driver was buggy in older qemu
1471     my $usbbus;
1472     if ($q35 || $arch eq 'aarch64') {
1473         $usbbus = 'ehci';
1474     } else {
1475         $usbbus = 'uhci';
1476     }
1477
1478     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1479 }
1480
1481 sub print_keyboarddevice_full {
1482     my ($conf, $arch) = @_;
1483
1484     return if $arch ne 'aarch64';
1485
1486     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1487 }
1488
1489 my sub get_drive_id {
1490     my ($drive) = @_;
1491     return "$drive->{interface}$drive->{index}";
1492 }
1493
1494 sub print_drivedevice_full {
1495     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1496
1497     my $device = '';
1498     my $maxdev = 0;
1499
1500     my $drive_id = get_drive_id($drive);
1501     if ($drive->{interface} eq 'virtio') {
1502         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1503         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1504         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1505     } elsif ($drive->{interface} eq 'scsi') {
1506
1507         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1508         my $unit = $drive->{index} % $maxdev;
1509         my $devicetype = 'hd';
1510         my $path = '';
1511         if (drive_is_cdrom($drive)) {
1512             $devicetype = 'cd';
1513         } else {
1514             if ($drive->{file} =~ m|^/|) {
1515                 $path = $drive->{file};
1516                 if (my $info = path_is_scsi($path)) {
1517                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1518                         $devicetype = 'block';
1519                     } elsif ($info->{type} == 1) { # tape
1520                         $devicetype = 'generic';
1521                     }
1522                 }
1523             } else {
1524                  $path = PVE::Storage::path($storecfg, $drive->{file});
1525             }
1526
1527             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1528             my $version = extract_version($machine_type, kvm_user_version());
1529             if ($path =~ m/^iscsi\:\/\// &&
1530                !min_version($version, 4, 1)) {
1531                 $devicetype = 'generic';
1532             }
1533         }
1534
1535         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1536             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1537         } else {
1538             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1539                 .",lun=$drive->{index}";
1540         }
1541         $device .= ",drive=drive-$drive_id,id=$drive_id";
1542
1543         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1544             $device .= ",rotation_rate=1";
1545         }
1546         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1547
1548     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1549         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1550         my $controller = int($drive->{index} / $maxdev);
1551         my $unit = $drive->{index} % $maxdev;
1552         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1553
1554         $device = "ide-$devicetype";
1555         if ($drive->{interface} eq 'ide') {
1556             $device .= ",bus=ide.$controller,unit=$unit";
1557         } else {
1558             $device .= ",bus=ahci$controller.$unit";
1559         }
1560         $device .= ",drive=drive-$drive_id,id=$drive_id";
1561
1562         if ($devicetype eq 'hd') {
1563             if (my $model = $drive->{model}) {
1564                 $model = URI::Escape::uri_unescape($model);
1565                 $device .= ",model=$model";
1566             }
1567             if ($drive->{ssd}) {
1568                 $device .= ",rotation_rate=1";
1569             }
1570         }
1571         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1572     } elsif ($drive->{interface} eq 'usb') {
1573         die "implement me";
1574         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1575     } else {
1576         die "unsupported interface type";
1577     }
1578
1579     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1580
1581     if (my $serial = $drive->{serial}) {
1582         $serial = URI::Escape::uri_unescape($serial);
1583         $device .= ",serial=$serial";
1584     }
1585
1586
1587     return $device;
1588 }
1589
1590 sub get_initiator_name {
1591     my $initiator;
1592
1593     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1594     while (defined(my $line = <$fh>)) {
1595         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1596         $initiator = $1;
1597         last;
1598     }
1599     $fh->close();
1600
1601     return $initiator;
1602 }
1603
1604 sub print_drive_commandline_full {
1605     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1606
1607     my $path;
1608     my $volid = $drive->{file};
1609     my $format = $drive->{format};
1610     my $drive_id = get_drive_id($drive);
1611
1612     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1613     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1614
1615     if (drive_is_cdrom($drive)) {
1616         $path = get_iso_path($storecfg, $vmid, $volid);
1617         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1618     } else {
1619         if ($storeid) {
1620             $path = PVE::Storage::path($storecfg, $volid);
1621             $format //= qemu_img_format($scfg, $volname);
1622         } else {
1623             $path = $volid;
1624             $format //= "raw";
1625         }
1626    }
1627
1628    my $is_rbd = $path =~ m/^rbd:/;
1629
1630     my $opts = '';
1631     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1632     foreach my $o (@qemu_drive_options) {
1633         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1634     }
1635
1636     # snapshot only accepts on|off
1637     if (defined($drive->{snapshot})) {
1638         my $v = $drive->{snapshot} ? 'on' : 'off';
1639         $opts .= ",snapshot=$v";
1640     }
1641
1642     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1643         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1644     }
1645
1646     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1647         my ($dir, $qmpname) = @$type;
1648         if (my $v = $drive->{"mbps$dir"}) {
1649             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1650         }
1651         if (my $v = $drive->{"mbps${dir}_max"}) {
1652             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1653         }
1654         if (my $v = $drive->{"bps${dir}_max_length"}) {
1655             $opts .= ",throttling.bps$qmpname-max-length=$v";
1656         }
1657         if (my $v = $drive->{"iops${dir}"}) {
1658             $opts .= ",throttling.iops$qmpname=$v";
1659         }
1660         if (my $v = $drive->{"iops${dir}_max"}) {
1661             $opts .= ",throttling.iops$qmpname-max=$v";
1662         }
1663         if (my $v = $drive->{"iops${dir}_max_length"}) {
1664             $opts .= ",throttling.iops$qmpname-max-length=$v";
1665         }
1666     }
1667
1668     if ($pbs_name) {
1669         $format = "rbd" if $is_rbd;
1670         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1671             if !$format;
1672         $opts .= ",format=alloc-track,file.driver=$format";
1673     } elsif ($format) {
1674         $opts .= ",format=$format";
1675     }
1676
1677     my $cache_direct = 0;
1678
1679     if (my $cache = $drive->{cache}) {
1680         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1681     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1682         $opts .= ",cache=none";
1683         $cache_direct = 1;
1684     }
1685
1686     # io_uring with cache mode writeback or writethrough on krbd will hang...
1687     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1688
1689     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1690     # sometimes, just plain disable...
1691     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1692
1693     # io_uring causes problems when used with CIFS since kernel 5.15
1694     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1695     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1696
1697     if (!$drive->{aio}) {
1698         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1699             # io_uring supports all cache modes
1700             $opts .= ",aio=io_uring";
1701         } else {
1702             # aio native works only with O_DIRECT
1703             if($cache_direct) {
1704                 $opts .= ",aio=native";
1705             } else {
1706                 $opts .= ",aio=threads";
1707             }
1708         }
1709     }
1710
1711     if (!drive_is_cdrom($drive)) {
1712         my $detectzeroes;
1713         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1714             $detectzeroes = 'off';
1715         } elsif ($drive->{discard}) {
1716             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1717         } else {
1718             # This used to be our default with discard not being specified:
1719             $detectzeroes = 'on';
1720         }
1721
1722         # note: 'detect-zeroes' works per blockdev and we want it to persist
1723         # after the alloc-track is removed, so put it on 'file' directly
1724         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1725         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1726     }
1727
1728     if ($pbs_name) {
1729         $opts .= ",backing=$pbs_name";
1730         $opts .= ",auto-remove=on";
1731     }
1732
1733     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1734     my $file_param = "file";
1735     if ($pbs_name) {
1736         # non-rbd drivers require the underlying file to be a seperate block
1737         # node, so add a second .file indirection
1738         $file_param .= ".file" if !$is_rbd;
1739         $file_param .= ".filename";
1740     }
1741     my $pathinfo = $path ? "$file_param=$path," : '';
1742
1743     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1744 }
1745
1746 sub print_pbs_blockdev {
1747     my ($pbs_conf, $pbs_name) = @_;
1748     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1749     $blockdev .= ",repository=$pbs_conf->{repository}";
1750     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1751     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1752     $blockdev .= ",archive=$pbs_conf->{archive}";
1753     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1754     return $blockdev;
1755 }
1756
1757 sub print_netdevice_full {
1758     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1759
1760     my $device = $net->{model};
1761     if ($net->{model} eq 'virtio') {
1762          $device = 'virtio-net-pci';
1763      };
1764
1765     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1766     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1767     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1768         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1769         # and out of each queue plus one config interrupt and control vector queue
1770         my $vectors = $net->{queues} * 2 + 2;
1771         $tmpstr .= ",vectors=$vectors,mq=on";
1772         if (min_version($machine_version, 7, 1)) {
1773             $tmpstr .= ",packed=on";
1774         }
1775     }
1776
1777     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1778         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1779     }
1780
1781     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1782
1783     if (my $mtu = $net->{mtu}) {
1784         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1785             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1786             if ($mtu == 1) {
1787                  $mtu = $bridge_mtu;
1788             } elsif ($mtu < 576) {
1789                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1790             } elsif ($mtu > $bridge_mtu) {
1791                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1792             }
1793             $tmpstr .= ",host_mtu=$mtu";
1794         } else {
1795             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1796         }
1797     }
1798
1799     if ($use_old_bios_files) {
1800         my $romfile;
1801         if ($device eq 'virtio-net-pci') {
1802             $romfile = 'pxe-virtio.rom';
1803         } elsif ($device eq 'e1000') {
1804             $romfile = 'pxe-e1000.rom';
1805         } elsif ($device eq 'e1000e') {
1806             $romfile = 'pxe-e1000e.rom';
1807         } elsif ($device eq 'ne2k') {
1808             $romfile = 'pxe-ne2k_pci.rom';
1809         } elsif ($device eq 'pcnet') {
1810             $romfile = 'pxe-pcnet.rom';
1811         } elsif ($device eq 'rtl8139') {
1812             $romfile = 'pxe-rtl8139.rom';
1813         }
1814         $tmpstr .= ",romfile=$romfile" if $romfile;
1815     }
1816
1817     return $tmpstr;
1818 }
1819
1820 sub print_netdev_full {
1821     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1822
1823     my $i = '';
1824     if ($netid =~ m/^net(\d+)$/) {
1825         $i = int($1);
1826     }
1827
1828     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1829
1830     my $ifname = "tap${vmid}i$i";
1831
1832     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1833     die "interface name '$ifname' is too long (max 15 character)\n"
1834         if length($ifname) >= 16;
1835
1836     my $vhostparam = '';
1837     if (is_native($arch)) {
1838         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1839     }
1840
1841     my $vmname = $conf->{name} || "vm$vmid";
1842
1843     my $netdev = "";
1844     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1845
1846     if ($net->{bridge}) {
1847         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1848             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1849     } else {
1850         $netdev = "type=user,id=$netid,hostname=$vmname";
1851     }
1852
1853     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1854
1855     return $netdev;
1856 }
1857
1858 my $vga_map = {
1859     'cirrus' => 'cirrus-vga',
1860     'std' => 'VGA',
1861     'vmware' => 'vmware-svga',
1862     'virtio' => 'virtio-vga',
1863     'virtio-gl' => 'virtio-vga-gl',
1864 };
1865
1866 sub print_vga_device {
1867     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1868
1869     my $type = $vga_map->{$vga->{type}};
1870     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1871         $type = 'virtio-gpu';
1872     }
1873     my $vgamem_mb = $vga->{memory};
1874
1875     my $max_outputs = '';
1876     if ($qxlnum) {
1877         $type = $id ? 'qxl' : 'qxl-vga';
1878
1879         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1880             # set max outputs so linux can have up to 4 qxl displays with one device
1881             if (min_version($machine_version, 4, 1)) {
1882                 $max_outputs = ",max_outputs=4";
1883             }
1884         }
1885     }
1886
1887     die "no devicetype for $vga->{type}\n" if !$type;
1888
1889     my $memory = "";
1890     if ($vgamem_mb) {
1891         if ($vga->{type} =~ /^virtio/) {
1892             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1893             $memory = ",max_hostmem=$bytes";
1894         } elsif ($qxlnum) {
1895             # from https://www.spice-space.org/multiple-monitors.html
1896             $memory = ",vgamem_mb=$vga->{memory}";
1897             my $ram = $vgamem_mb * 4;
1898             my $vram = $vgamem_mb * 2;
1899             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1900         } else {
1901             $memory = ",vgamem_mb=$vga->{memory}";
1902         }
1903     } elsif ($qxlnum && $id) {
1904         $memory = ",ram_size=67108864,vram_size=33554432";
1905     }
1906
1907     my $edidoff = "";
1908     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1909         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1910     }
1911
1912     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1913     my $vgaid = "vga" . ($id // '');
1914     my $pciaddr;
1915     if ($q35 && $vgaid eq 'vga') {
1916         # the first display uses pcie.0 bus on q35 machines
1917         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1918     } else {
1919         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1920     }
1921
1922     if ($vga->{type} eq 'virtio-gl') {
1923         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1924         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1925             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1926
1927         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1928             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1929     }
1930
1931     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1932 }
1933
1934 sub parse_number_sets {
1935     my ($set) = @_;
1936     my $res = [];
1937     foreach my $part (split(/;/, $set)) {
1938         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1939             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1940             push @$res, [ $1, $2 ];
1941         } else {
1942             die "invalid range: $part\n";
1943         }
1944     }
1945     return $res;
1946 }
1947
1948 sub parse_numa {
1949     my ($data) = @_;
1950
1951     my $res = parse_property_string($numa_fmt, $data);
1952     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1953     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1954     return $res;
1955 }
1956
1957 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1958 sub parse_net {
1959     my ($data, $disable_mac_autogen) = @_;
1960
1961     my $res = eval { parse_property_string($net_fmt, $data) };
1962     if ($@) {
1963         warn $@;
1964         return;
1965     }
1966     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1967         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1968         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1969     }
1970     return $res;
1971 }
1972
1973 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1974 sub parse_ipconfig {
1975     my ($data) = @_;
1976
1977     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1978     if ($@) {
1979         warn $@;
1980         return;
1981     }
1982
1983     if ($res->{gw} && !$res->{ip}) {
1984         warn 'gateway specified without specifying an IP address';
1985         return;
1986     }
1987     if ($res->{gw6} && !$res->{ip6}) {
1988         warn 'IPv6 gateway specified without specifying an IPv6 address';
1989         return;
1990     }
1991     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1992         warn 'gateway specified together with DHCP';
1993         return;
1994     }
1995     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1996         # gw6 + auto/dhcp
1997         warn "IPv6 gateway specified together with $res->{ip6} address";
1998         return;
1999     }
2000
2001     if (!$res->{ip} && !$res->{ip6}) {
2002         return { ip => 'dhcp', ip6 => 'dhcp' };
2003     }
2004
2005     return $res;
2006 }
2007
2008 sub print_net {
2009     my $net = shift;
2010
2011     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2012 }
2013
2014 sub add_random_macs {
2015     my ($settings) = @_;
2016
2017     foreach my $opt (keys %$settings) {
2018         next if $opt !~ m/^net(\d+)$/;
2019         my $net = parse_net($settings->{$opt});
2020         next if !$net;
2021         $settings->{$opt} = print_net($net);
2022     }
2023 }
2024
2025 sub vm_is_volid_owner {
2026     my ($storecfg, $vmid, $volid) = @_;
2027
2028     if ($volid !~  m|^/|) {
2029         my ($path, $owner);
2030         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2031         if ($owner && ($owner == $vmid)) {
2032             return 1;
2033         }
2034     }
2035
2036     return;
2037 }
2038
2039 sub vmconfig_register_unused_drive {
2040     my ($storecfg, $vmid, $conf, $drive) = @_;
2041
2042     if (drive_is_cloudinit($drive)) {
2043         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2044         warn $@ if $@;
2045         delete $conf->{cloudinit};
2046     } elsif (!drive_is_cdrom($drive)) {
2047         my $volid = $drive->{file};
2048         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2049             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2050         }
2051     }
2052 }
2053
2054 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2055 my $smbios1_fmt = {
2056     uuid => {
2057         type => 'string',
2058         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2059         format_description => 'UUID',
2060         description => "Set SMBIOS1 UUID.",
2061         optional => 1,
2062     },
2063     version => {
2064         type => 'string',
2065         pattern => '[A-Za-z0-9+\/]+={0,2}',
2066         format_description => 'Base64 encoded string',
2067         description => "Set SMBIOS1 version.",
2068         optional => 1,
2069     },
2070     serial => {
2071         type => 'string',
2072         pattern => '[A-Za-z0-9+\/]+={0,2}',
2073         format_description => 'Base64 encoded string',
2074         description => "Set SMBIOS1 serial number.",
2075         optional => 1,
2076     },
2077     manufacturer => {
2078         type => 'string',
2079         pattern => '[A-Za-z0-9+\/]+={0,2}',
2080         format_description => 'Base64 encoded string',
2081         description => "Set SMBIOS1 manufacturer.",
2082         optional => 1,
2083     },
2084     product => {
2085         type => 'string',
2086         pattern => '[A-Za-z0-9+\/]+={0,2}',
2087         format_description => 'Base64 encoded string',
2088         description => "Set SMBIOS1 product ID.",
2089         optional => 1,
2090     },
2091     sku => {
2092         type => 'string',
2093         pattern => '[A-Za-z0-9+\/]+={0,2}',
2094         format_description => 'Base64 encoded string',
2095         description => "Set SMBIOS1 SKU string.",
2096         optional => 1,
2097     },
2098     family => {
2099         type => 'string',
2100         pattern => '[A-Za-z0-9+\/]+={0,2}',
2101         format_description => 'Base64 encoded string',
2102         description => "Set SMBIOS1 family string.",
2103         optional => 1,
2104     },
2105     base64 => {
2106         type => 'boolean',
2107         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2108         optional => 1,
2109     },
2110 };
2111
2112 sub parse_smbios1 {
2113     my ($data) = @_;
2114
2115     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2116     warn $@ if $@;
2117     return $res;
2118 }
2119
2120 sub print_smbios1 {
2121     my ($smbios1) = @_;
2122     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2123 }
2124
2125 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2126
2127 sub parse_watchdog {
2128     my ($value) = @_;
2129
2130     return if !$value;
2131
2132     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2133     warn $@ if $@;
2134     return $res;
2135 }
2136
2137 sub parse_guest_agent {
2138     my ($conf) = @_;
2139
2140     return {} if !defined($conf->{agent});
2141
2142     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2143     warn $@ if $@;
2144
2145     # if the agent is disabled ignore the other potentially set properties
2146     return {} if !$res->{enabled};
2147     return $res;
2148 }
2149
2150 sub get_qga_key {
2151     my ($conf, $key) = @_;
2152     return undef if !defined($conf->{agent});
2153
2154     my $agent = parse_guest_agent($conf);
2155     return $agent->{$key};
2156 }
2157
2158 sub parse_vga {
2159     my ($value) = @_;
2160
2161     return {} if !$value;
2162     my $res = eval { parse_property_string($vga_fmt, $value) };
2163     warn $@ if $@;
2164     return $res;
2165 }
2166
2167 sub parse_rng {
2168     my ($value) = @_;
2169
2170     return if !$value;
2171
2172     my $res = eval { parse_property_string($rng_fmt, $value) };
2173     warn $@ if $@;
2174     return $res;
2175 }
2176
2177 sub parse_meta_info {
2178     my ($value) = @_;
2179
2180     return if !$value;
2181
2182     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2183     warn $@ if $@;
2184     return $res;
2185 }
2186
2187 sub new_meta_info_string {
2188     my () = @_; # for now do not allow to override any value
2189
2190     return PVE::JSONSchema::print_property_string(
2191         {
2192             'creation-qemu' => kvm_user_version(),
2193             ctime => "". int(time()),
2194         },
2195         $meta_info_fmt
2196     );
2197 }
2198
2199 sub qemu_created_version_fixups {
2200     my ($conf, $forcemachine, $kvmver) = @_;
2201
2202     my $meta = parse_meta_info($conf->{meta}) // {};
2203     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2204
2205     # check if we need to apply some handling for VMs that always use the latest machine version but
2206     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2207     # would be required (we do not want to pin machine version for non-windows OS type)
2208     if (
2209         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2210         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2211         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2212         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2213     ) {
2214         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2215         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2216             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2217             # and thus with the predictable interface naming of systemd
2218             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2219         }
2220     }
2221     return;
2222 }
2223
2224 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2225 sub verify_usb_device {
2226     my ($value, $noerr) = @_;
2227
2228     return $value if parse_usb_device($value);
2229
2230     return if $noerr;
2231
2232     die "unable to parse usb device\n";
2233 }
2234
2235 # add JSON properties for create and set function
2236 sub json_config_properties {
2237     my ($prop, $with_disk_alloc) = @_;
2238
2239     my $skip_json_config_opts = {
2240         parent => 1,
2241         snaptime => 1,
2242         vmstate => 1,
2243         runningmachine => 1,
2244         runningcpu => 1,
2245         meta => 1,
2246     };
2247
2248     foreach my $opt (keys %$confdesc) {
2249         next if $skip_json_config_opts->{$opt};
2250
2251         if ($with_disk_alloc && is_valid_drivename($opt)) {
2252             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2253         } else {
2254             $prop->{$opt} = $confdesc->{$opt};
2255         }
2256     }
2257
2258     return $prop;
2259 }
2260
2261 # Properties that we can read from an OVF file
2262 sub json_ovf_properties {
2263     my $prop = {};
2264
2265     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2266         $prop->{$device} = {
2267             type => 'string',
2268             format => 'pve-volume-id-or-absolute-path',
2269             description => "Disk image that gets imported to $device",
2270             optional => 1,
2271         };
2272     }
2273
2274     $prop->{cores} = {
2275         type => 'integer',
2276         description => "The number of CPU cores.",
2277         optional => 1,
2278     };
2279     $prop->{memory} = {
2280         type => 'integer',
2281         description => "Amount of RAM for the VM in MB.",
2282         optional => 1,
2283     };
2284     $prop->{name} = {
2285         type => 'string',
2286         description => "Name of the VM.",
2287         optional => 1,
2288     };
2289
2290     return $prop;
2291 }
2292
2293 # return copy of $confdesc_cloudinit to generate documentation
2294 sub cloudinit_config_properties {
2295
2296     return dclone($confdesc_cloudinit);
2297 }
2298
2299 sub cloudinit_pending_properties {
2300     my $p = {
2301         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2302         name => 1,
2303     };
2304     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2305     return $p;
2306 }
2307
2308 sub check_type {
2309     my ($key, $value) = @_;
2310
2311     die "unknown setting '$key'\n" if !$confdesc->{$key};
2312
2313     my $type = $confdesc->{$key}->{type};
2314
2315     if (!defined($value)) {
2316         die "got undefined value\n";
2317     }
2318
2319     if ($value =~ m/[\n\r]/) {
2320         die "property contains a line feed\n";
2321     }
2322
2323     if ($type eq 'boolean') {
2324         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2325         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2326         die "type check ('boolean') failed - got '$value'\n";
2327     } elsif ($type eq 'integer') {
2328         return int($1) if $value =~ m/^(\d+)$/;
2329         die "type check ('integer') failed - got '$value'\n";
2330     } elsif ($type eq 'number') {
2331         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2332         die "type check ('number') failed - got '$value'\n";
2333     } elsif ($type eq 'string') {
2334         if (my $fmt = $confdesc->{$key}->{format}) {
2335             PVE::JSONSchema::check_format($fmt, $value);
2336             return $value;
2337         }
2338         $value =~ s/^\"(.*)\"$/$1/;
2339         return $value;
2340     } else {
2341         die "internal error"
2342     }
2343 }
2344
2345 sub destroy_vm {
2346     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2347
2348     my $conf = PVE::QemuConfig->load_config($vmid);
2349
2350     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2351         PVE::QemuConfig->check_lock($conf);
2352     }
2353
2354     if ($conf->{template}) {
2355         # check if any base image is still used by a linked clone
2356         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2357                 my ($ds, $drive) = @_;
2358                 return if drive_is_cdrom($drive);
2359
2360                 my $volid = $drive->{file};
2361                 return if !$volid || $volid =~ m|^/|;
2362
2363                 die "base volume '$volid' is still in use by linked cloned\n"
2364                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2365
2366         });
2367     }
2368
2369     my $volids = {};
2370     my $remove_owned_drive = sub {
2371         my ($ds, $drive) = @_;
2372         return if drive_is_cdrom($drive, 1);
2373
2374         my $volid = $drive->{file};
2375         return if !$volid || $volid =~ m|^/|;
2376         return if $volids->{$volid};
2377
2378         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2379         return if !$path || !$owner || ($owner != $vmid);
2380
2381         $volids->{$volid} = 1;
2382         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2383         warn "Could not remove disk '$volid', check manually: $@" if $@;
2384     };
2385
2386     # only remove disks owned by this VM (referenced in the config)
2387     my $include_opts = {
2388         include_unused => 1,
2389         extra_keys => ['vmstate'],
2390     };
2391     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2392
2393     for my $snap (values %{$conf->{snapshots}}) {
2394         next if !defined($snap->{vmstate});
2395         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2396         next if !defined($drive);
2397         $remove_owned_drive->('vmstate', $drive);
2398     }
2399
2400     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2401
2402     if ($purge_unreferenced) { # also remove unreferenced disk
2403         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2404         PVE::Storage::foreach_volid($vmdisks, sub {
2405             my ($volid, $sid, $volname, $d) = @_;
2406             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2407             warn $@ if $@;
2408         });
2409     }
2410
2411     if (defined $replacement_conf) {
2412         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2413     } else {
2414         PVE::QemuConfig->destroy_config($vmid);
2415     }
2416 }
2417
2418 sub parse_vm_config {
2419     my ($filename, $raw, $strict) = @_;
2420
2421     return if !defined($raw);
2422
2423     my $res = {
2424         digest => Digest::SHA::sha1_hex($raw),
2425         snapshots => {},
2426         pending => {},
2427         cloudinit => {},
2428     };
2429
2430     my $handle_error = sub {
2431         my ($msg) = @_;
2432
2433         if ($strict) {
2434             die $msg;
2435         } else {
2436             warn $msg;
2437         }
2438     };
2439
2440     $filename =~ m|/qemu-server/(\d+)\.conf$|
2441         || die "got strange filename '$filename'";
2442
2443     my $vmid = $1;
2444
2445     my $conf = $res;
2446     my $descr;
2447     my $finish_description = sub {
2448         if (defined($descr)) {
2449             $descr =~ s/\s+$//;
2450             $conf->{description} = $descr;
2451         }
2452         $descr = undef;
2453     };
2454     my $section = '';
2455
2456     my @lines = split(/\n/, $raw);
2457     foreach my $line (@lines) {
2458         next if $line =~ m/^\s*$/;
2459
2460         if ($line =~ m/^\[PENDING\]\s*$/i) {
2461             $section = 'pending';
2462             $finish_description->();
2463             $conf = $res->{$section} = {};
2464             next;
2465         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2466             $section = 'cloudinit';
2467             $finish_description->();
2468             $conf = $res->{$section} = {};
2469             next;
2470
2471         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2472             $section = $1;
2473             $finish_description->();
2474             $conf = $res->{snapshots}->{$section} = {};
2475             next;
2476         }
2477
2478         if ($line =~ m/^\#(.*)$/) {
2479             $descr = '' if !defined($descr);
2480             $descr .= PVE::Tools::decode_text($1) . "\n";
2481             next;
2482         }
2483
2484         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2485             $descr = '' if !defined($descr);
2486             $descr .= PVE::Tools::decode_text($2);
2487         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2488             $conf->{snapstate} = $1;
2489         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2490             my $key = $1;
2491             my $value = $2;
2492             $conf->{$key} = $value;
2493         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2494             my $value = $1;
2495             if ($section eq 'pending') {
2496                 $conf->{delete} = $value; # we parse this later
2497             } else {
2498                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2499             }
2500         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2501             my $key = $1;
2502             my $value = $2;
2503             if ($section eq 'cloudinit') {
2504                 # ignore validation only used for informative purpose
2505                 $conf->{$key} = $value;
2506                 next;
2507             }
2508             eval { $value = check_type($key, $value); };
2509             if ($@) {
2510                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2511             } else {
2512                 $key = 'ide2' if $key eq 'cdrom';
2513                 my $fmt = $confdesc->{$key}->{format};
2514                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2515                     my $v = parse_drive($key, $value);
2516                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2517                         $v->{file} = $volid;
2518                         $value = print_drive($v);
2519                     } else {
2520                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2521                         next;
2522                     }
2523                 }
2524
2525                 $conf->{$key} = $value;
2526             }
2527         } else {
2528             $handle_error->("vm $vmid - unable to parse config: $line\n");
2529         }
2530     }
2531
2532     $finish_description->();
2533     delete $res->{snapstate}; # just to be sure
2534
2535     return $res;
2536 }
2537
2538 sub write_vm_config {
2539     my ($filename, $conf) = @_;
2540
2541     delete $conf->{snapstate}; # just to be sure
2542
2543     if ($conf->{cdrom}) {
2544         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2545         $conf->{ide2} = $conf->{cdrom};
2546         delete $conf->{cdrom};
2547     }
2548
2549     # we do not use 'smp' any longer
2550     if ($conf->{sockets}) {
2551         delete $conf->{smp};
2552     } elsif ($conf->{smp}) {
2553         $conf->{sockets} = $conf->{smp};
2554         delete $conf->{cores};
2555         delete $conf->{smp};
2556     }
2557
2558     my $used_volids = {};
2559
2560     my $cleanup_config = sub {
2561         my ($cref, $pending, $snapname) = @_;
2562
2563         foreach my $key (keys %$cref) {
2564             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2565                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2566             my $value = $cref->{$key};
2567             if ($key eq 'delete') {
2568                 die "propertry 'delete' is only allowed in [PENDING]\n"
2569                     if !$pending;
2570                 # fixme: check syntax?
2571                 next;
2572             }
2573             eval { $value = check_type($key, $value); };
2574             die "unable to parse value of '$key' - $@" if $@;
2575
2576             $cref->{$key} = $value;
2577
2578             if (!$snapname && is_valid_drivename($key)) {
2579                 my $drive = parse_drive($key, $value);
2580                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2581             }
2582         }
2583     };
2584
2585     &$cleanup_config($conf);
2586
2587     &$cleanup_config($conf->{pending}, 1);
2588
2589     foreach my $snapname (keys %{$conf->{snapshots}}) {
2590         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2591         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2592     }
2593
2594     # remove 'unusedX' settings if we re-add a volume
2595     foreach my $key (keys %$conf) {
2596         my $value = $conf->{$key};
2597         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2598             delete $conf->{$key};
2599         }
2600     }
2601
2602     my $generate_raw_config = sub {
2603         my ($conf, $pending) = @_;
2604
2605         my $raw = '';
2606
2607         # add description as comment to top of file
2608         if (defined(my $descr = $conf->{description})) {
2609             if ($descr) {
2610                 foreach my $cl (split(/\n/, $descr)) {
2611                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2612                 }
2613             } else {
2614                 $raw .= "#\n" if $pending;
2615             }
2616         }
2617
2618         foreach my $key (sort keys %$conf) {
2619             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2620             $raw .= "$key: $conf->{$key}\n";
2621         }
2622         return $raw;
2623     };
2624
2625     my $raw = &$generate_raw_config($conf);
2626
2627     if (scalar(keys %{$conf->{pending}})){
2628         $raw .= "\n[PENDING]\n";
2629         $raw .= &$generate_raw_config($conf->{pending}, 1);
2630     }
2631
2632     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2633         $raw .= "\n[special:cloudinit]\n";
2634         $raw .= &$generate_raw_config($conf->{cloudinit});
2635     }
2636
2637     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2638         $raw .= "\n[$snapname]\n";
2639         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2640     }
2641
2642     return $raw;
2643 }
2644
2645 sub load_defaults {
2646
2647     my $res = {};
2648
2649     # we use static defaults from our JSON schema configuration
2650     foreach my $key (keys %$confdesc) {
2651         if (defined(my $default = $confdesc->{$key}->{default})) {
2652             $res->{$key} = $default;
2653         }
2654     }
2655
2656     return $res;
2657 }
2658
2659 sub config_list {
2660     my $vmlist = PVE::Cluster::get_vmlist();
2661     my $res = {};
2662     return $res if !$vmlist || !$vmlist->{ids};
2663     my $ids = $vmlist->{ids};
2664     my $nodename = nodename();
2665
2666     foreach my $vmid (keys %$ids) {
2667         my $d = $ids->{$vmid};
2668         next if !$d->{node} || $d->{node} ne $nodename;
2669         next if !$d->{type} || $d->{type} ne 'qemu';
2670         $res->{$vmid}->{exists} = 1;
2671     }
2672     return $res;
2673 }
2674
2675 # test if VM uses local resources (to prevent migration)
2676 sub check_local_resources {
2677     my ($conf, $noerr) = @_;
2678
2679     my @loc_res = ();
2680
2681     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2682     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2683
2684     push @loc_res, "ivshmem" if $conf->{ivshmem};
2685
2686     foreach my $k (keys %$conf) {
2687         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2688         # sockets are safe: they will recreated be on the target side post-migrate
2689         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2690         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2691     }
2692
2693     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2694
2695     return \@loc_res;
2696 }
2697
2698 # check if used storages are available on all nodes (use by migrate)
2699 sub check_storage_availability {
2700     my ($storecfg, $conf, $node) = @_;
2701
2702     PVE::QemuConfig->foreach_volume($conf, sub {
2703         my ($ds, $drive) = @_;
2704
2705         my $volid = $drive->{file};
2706         return if !$volid;
2707
2708         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2709         return if !$sid;
2710
2711         # check if storage is available on both nodes
2712         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2713         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2714
2715         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2716
2717         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2718             if !$scfg->{content}->{$vtype};
2719    });
2720 }
2721
2722 # list nodes where all VM images are available (used by has_feature API)
2723 sub shared_nodes {
2724     my ($conf, $storecfg) = @_;
2725
2726     my $nodelist = PVE::Cluster::get_nodelist();
2727     my $nodehash = { map { $_ => 1 } @$nodelist };
2728     my $nodename = nodename();
2729
2730     PVE::QemuConfig->foreach_volume($conf, sub {
2731         my ($ds, $drive) = @_;
2732
2733         my $volid = $drive->{file};
2734         return if !$volid;
2735
2736         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2737         if ($storeid) {
2738             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2739             if ($scfg->{disable}) {
2740                 $nodehash = {};
2741             } elsif (my $avail = $scfg->{nodes}) {
2742                 foreach my $node (keys %$nodehash) {
2743                     delete $nodehash->{$node} if !$avail->{$node};
2744                 }
2745             } elsif (!$scfg->{shared}) {
2746                 foreach my $node (keys %$nodehash) {
2747                     delete $nodehash->{$node} if $node ne $nodename
2748                 }
2749             }
2750         }
2751     });
2752
2753     return $nodehash
2754 }
2755
2756 sub check_local_storage_availability {
2757     my ($conf, $storecfg) = @_;
2758
2759     my $nodelist = PVE::Cluster::get_nodelist();
2760     my $nodehash = { map { $_ => {} } @$nodelist };
2761
2762     PVE::QemuConfig->foreach_volume($conf, sub {
2763         my ($ds, $drive) = @_;
2764
2765         my $volid = $drive->{file};
2766         return if !$volid;
2767
2768         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2769         if ($storeid) {
2770             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2771
2772             if ($scfg->{disable}) {
2773                 foreach my $node (keys %$nodehash) {
2774                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2775                 }
2776             } elsif (my $avail = $scfg->{nodes}) {
2777                 foreach my $node (keys %$nodehash) {
2778                     if (!$avail->{$node}) {
2779                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2780                     }
2781                 }
2782             }
2783         }
2784     });
2785
2786     foreach my $node (values %$nodehash) {
2787         if (my $unavail = $node->{unavailable_storages}) {
2788             $node->{unavailable_storages} = [ sort keys %$unavail ];
2789         }
2790     }
2791
2792     return $nodehash
2793 }
2794
2795 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2796 sub check_running {
2797     my ($vmid, $nocheck, $node) = @_;
2798
2799     # $nocheck is set when called during a migration, in which case the config
2800     # file might still or already reside on the *other* node
2801     # - because rename has already happened, and current node is source
2802     # - because rename hasn't happened yet, and current node is target
2803     # - because rename has happened, current node is target, but hasn't yet
2804     # processed it yet
2805     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2806     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2807 }
2808
2809 sub vzlist {
2810
2811     my $vzlist = config_list();
2812
2813     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2814
2815     while (defined(my $de = $fd->read)) {
2816         next if $de !~ m/^(\d+)\.pid$/;
2817         my $vmid = $1;
2818         next if !defined($vzlist->{$vmid});
2819         if (my $pid = check_running($vmid)) {
2820             $vzlist->{$vmid}->{pid} = $pid;
2821         }
2822     }
2823
2824     return $vzlist;
2825 }
2826
2827 our $vmstatus_return_properties = {
2828     vmid => get_standard_option('pve-vmid'),
2829     status => {
2830         description => "QEMU process status.",
2831         type => 'string',
2832         enum => ['stopped', 'running'],
2833     },
2834     maxmem => {
2835         description => "Maximum memory in bytes.",
2836         type => 'integer',
2837         optional => 1,
2838         renderer => 'bytes',
2839     },
2840     maxdisk => {
2841         description => "Root disk size in bytes.",
2842         type => 'integer',
2843         optional => 1,
2844         renderer => 'bytes',
2845     },
2846     name => {
2847         description => "VM name.",
2848         type => 'string',
2849         optional => 1,
2850     },
2851     qmpstatus => {
2852         description => "QEMU QMP agent status.",
2853         type => 'string',
2854         optional => 1,
2855     },
2856     pid => {
2857         description => "PID of running qemu process.",
2858         type => 'integer',
2859         optional => 1,
2860     },
2861     uptime => {
2862         description => "Uptime.",
2863         type => 'integer',
2864         optional => 1,
2865         renderer => 'duration',
2866     },
2867     cpus => {
2868         description => "Maximum usable CPUs.",
2869         type => 'number',
2870         optional => 1,
2871     },
2872     lock => {
2873         description => "The current config lock, if any.",
2874         type => 'string',
2875         optional => 1,
2876     },
2877     tags => {
2878         description  => "The current configured tags, if any",
2879         type => 'string',
2880         optional => 1,
2881     },
2882     'running-machine' => {
2883         description => "The currently running machine type (if running).",
2884         type => 'string',
2885         optional => 1,
2886     },
2887     'running-qemu' => {
2888         description => "The currently running QEMU version (if running).",
2889         type => 'string',
2890         optional => 1,
2891     },
2892 };
2893
2894 my $last_proc_pid_stat;
2895
2896 # get VM status information
2897 # This must be fast and should not block ($full == false)
2898 # We only query KVM using QMP if $full == true (this can be slow)
2899 sub vmstatus {
2900     my ($opt_vmid, $full) = @_;
2901
2902     my $res = {};
2903
2904     my $storecfg = PVE::Storage::config();
2905
2906     my $list = vzlist();
2907     my $defaults = load_defaults();
2908
2909     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2910
2911     my $cpucount = $cpuinfo->{cpus} || 1;
2912
2913     foreach my $vmid (keys %$list) {
2914         next if $opt_vmid && ($vmid ne $opt_vmid);
2915
2916         my $conf = PVE::QemuConfig->load_config($vmid);
2917
2918         my $d = { vmid => int($vmid) };
2919         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2920
2921         # fixme: better status?
2922         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2923
2924         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2925         if (defined($size)) {
2926             $d->{disk} = 0; # no info available
2927             $d->{maxdisk} = $size;
2928         } else {
2929             $d->{disk} = 0;
2930             $d->{maxdisk} = 0;
2931         }
2932
2933         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2934             * ($conf->{cores} || $defaults->{cores});
2935         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2936         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2937
2938         $d->{name} = $conf->{name} || "VM $vmid";
2939         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2940             : $defaults->{memory}*(1024*1024);
2941
2942         if ($conf->{balloon}) {
2943             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2944             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2945                 : $defaults->{shares};
2946         }
2947
2948         $d->{uptime} = 0;
2949         $d->{cpu} = 0;
2950         $d->{mem} = 0;
2951
2952         $d->{netout} = 0;
2953         $d->{netin} = 0;
2954
2955         $d->{diskread} = 0;
2956         $d->{diskwrite} = 0;
2957
2958         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2959
2960         $d->{serial} = 1 if conf_has_serial($conf);
2961         $d->{lock} = $conf->{lock} if $conf->{lock};
2962         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2963
2964         $res->{$vmid} = $d;
2965     }
2966
2967     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2968     foreach my $dev (keys %$netdev) {
2969         next if $dev !~ m/^tap([1-9]\d*)i/;
2970         my $vmid = $1;
2971         my $d = $res->{$vmid};
2972         next if !$d;
2973
2974         $d->{netout} += $netdev->{$dev}->{receive};
2975         $d->{netin} += $netdev->{$dev}->{transmit};
2976
2977         if ($full) {
2978             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2979             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2980         }
2981
2982     }
2983
2984     my $ctime = gettimeofday;
2985
2986     foreach my $vmid (keys %$list) {
2987
2988         my $d = $res->{$vmid};
2989         my $pid = $d->{pid};
2990         next if !$pid;
2991
2992         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2993         next if !$pstat; # not running
2994
2995         my $used = $pstat->{utime} + $pstat->{stime};
2996
2997         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2998
2999         if ($pstat->{vsize}) {
3000             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3001         }
3002
3003         my $old = $last_proc_pid_stat->{$pid};
3004         if (!$old) {
3005             $last_proc_pid_stat->{$pid} = {
3006                 time => $ctime,
3007                 used => $used,
3008                 cpu => 0,
3009             };
3010             next;
3011         }
3012
3013         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3014
3015         if ($dtime > 1000) {
3016             my $dutime = $used -  $old->{used};
3017
3018             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3019             $last_proc_pid_stat->{$pid} = {
3020                 time => $ctime,
3021                 used => $used,
3022                 cpu => $d->{cpu},
3023             };
3024         } else {
3025             $d->{cpu} = $old->{cpu};
3026         }
3027     }
3028
3029     return $res if !$full;
3030
3031     my $qmpclient = PVE::QMPClient->new();
3032
3033     my $ballooncb = sub {
3034         my ($vmid, $resp) = @_;
3035
3036         my $info = $resp->{'return'};
3037         return if !$info->{max_mem};
3038
3039         my $d = $res->{$vmid};
3040
3041         # use memory assigned to VM
3042         $d->{maxmem} = $info->{max_mem};
3043         $d->{balloon} = $info->{actual};
3044
3045         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3046             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3047             $d->{freemem} = $info->{free_mem};
3048         }
3049
3050         $d->{ballooninfo} = $info;
3051     };
3052
3053     my $blockstatscb = sub {
3054         my ($vmid, $resp) = @_;
3055         my $data = $resp->{'return'} || [];
3056         my $totalrdbytes = 0;
3057         my $totalwrbytes = 0;
3058
3059         for my $blockstat (@$data) {
3060             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3061             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3062
3063             $blockstat->{device} =~ s/drive-//;
3064             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3065         }
3066         $res->{$vmid}->{diskread} = $totalrdbytes;
3067         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3068     };
3069
3070     my $machinecb = sub {
3071         my ($vmid, $resp) = @_;
3072         my $data = $resp->{'return'} || [];
3073
3074         $res->{$vmid}->{'running-machine'} =
3075             PVE::QemuServer::Machine::current_from_query_machines($data);
3076     };
3077
3078     my $versioncb = sub {
3079         my ($vmid, $resp) = @_;
3080         my $data = $resp->{'return'} // {};
3081         my $version = 'unknown';
3082
3083         if (my $v = $data->{qemu}) {
3084             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3085         }
3086
3087         $res->{$vmid}->{'running-qemu'} = $version;
3088     };
3089
3090     my $statuscb = sub {
3091         my ($vmid, $resp) = @_;
3092
3093         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3094         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3095         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3096         # this fails if ballon driver is not loaded, so this must be
3097         # the last commnand (following command are aborted if this fails).
3098         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3099
3100         my $status = 'unknown';
3101         if (!defined($status = $resp->{'return'}->{status})) {
3102             warn "unable to get VM status\n";
3103             return;
3104         }
3105
3106         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3107     };
3108
3109     foreach my $vmid (keys %$list) {
3110         next if $opt_vmid && ($vmid ne $opt_vmid);
3111         next if !$res->{$vmid}->{pid}; # not running
3112         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3113     }
3114
3115     $qmpclient->queue_execute(undef, 2);
3116
3117     foreach my $vmid (keys %$list) {
3118         next if $opt_vmid && ($vmid ne $opt_vmid);
3119         next if !$res->{$vmid}->{pid}; #not running
3120
3121         # we can't use the $qmpclient since it might have already aborted on
3122         # 'query-balloon', but this might also fail for older versions...
3123         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3124         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3125     }
3126
3127     foreach my $vmid (keys %$list) {
3128         next if $opt_vmid && ($vmid ne $opt_vmid);
3129         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3130     }
3131
3132     return $res;
3133 }
3134
3135 sub conf_has_serial {
3136     my ($conf) = @_;
3137
3138     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3139         if ($conf->{"serial$i"}) {
3140             return 1;
3141         }
3142     }
3143
3144     return 0;
3145 }
3146
3147 sub conf_has_audio {
3148     my ($conf, $id) = @_;
3149
3150     $id //= 0;
3151     my $audio = $conf->{"audio$id"};
3152     return if !defined($audio);
3153
3154     my $audioproperties = parse_property_string($audio_fmt, $audio);
3155     my $audiodriver = $audioproperties->{driver} // 'spice';
3156
3157     return {
3158         dev => $audioproperties->{device},
3159         dev_id => "audiodev$id",
3160         backend => $audiodriver,
3161         backend_id => "$audiodriver-backend${id}",
3162     };
3163 }
3164
3165 sub audio_devs {
3166     my ($audio, $audiopciaddr, $machine_version) = @_;
3167
3168     my $devs = [];
3169
3170     my $id = $audio->{dev_id};
3171     my $audiodev = "";
3172     if (min_version($machine_version, 4, 2)) {
3173         $audiodev = ",audiodev=$audio->{backend_id}";
3174     }
3175
3176     if ($audio->{dev} eq 'AC97') {
3177         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3178     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3179         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3180         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3181         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3182     } else {
3183         die "unkown audio device '$audio->{dev}', implement me!";
3184     }
3185
3186     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3187
3188     return $devs;
3189 }
3190
3191 sub get_tpm_paths {
3192     my ($vmid) = @_;
3193     return {
3194         socket => "/var/run/qemu-server/$vmid.swtpm",
3195         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3196     };
3197 }
3198
3199 sub add_tpm_device {
3200     my ($vmid, $devices, $conf) = @_;
3201
3202     return if !$conf->{tpmstate0};
3203
3204     my $paths = get_tpm_paths($vmid);
3205
3206     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3207     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3208     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3209 }
3210
3211 sub start_swtpm {
3212     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3213
3214     return if !$tpmdrive;
3215
3216     my $state;
3217     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3218     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3219     if ($storeid) {
3220         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3221     } else {
3222         $state = $tpm->{file};
3223     }
3224
3225     my $paths = get_tpm_paths($vmid);
3226
3227     # during migration, we will get state from remote
3228     #
3229     if (!$migration) {
3230         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3231         my $setup_cmd = [
3232             "swtpm_setup",
3233             "--tpmstate",
3234             "file://$state",
3235             "--createek",
3236             "--create-ek-cert",
3237             "--create-platform-cert",
3238             "--lock-nvram",
3239             "--config",
3240             "/etc/swtpm_setup.conf", # do not use XDG configs
3241             "--runas",
3242             "0", # force creation as root, error if not possible
3243             "--not-overwrite", # ignore existing state, do not modify
3244         ];
3245
3246         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3247         # TPM 2.0 supports ECC crypto, use if possible
3248         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3249
3250         run_command($setup_cmd, outfunc => sub {
3251             print "swtpm_setup: $1\n";
3252         });
3253     }
3254
3255     my $emulator_cmd = [
3256         "swtpm",
3257         "socket",
3258         "--tpmstate",
3259         "backend-uri=file://$state,mode=0600",
3260         "--ctrl",
3261         "type=unixio,path=$paths->{socket},mode=0600",
3262         "--pid",
3263         "file=$paths->{pid}",
3264         "--terminate", # terminate on QEMU disconnect
3265         "--daemon",
3266     ];
3267     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3268     run_command($emulator_cmd, outfunc => sub { print $1; });
3269
3270     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3271     while (! -e $paths->{pid}) {
3272         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3273         usleep(50_000);
3274     }
3275
3276     # return untainted PID of swtpm daemon so it can be killed on error
3277     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3278     return $1;
3279 }
3280
3281 sub vga_conf_has_spice {
3282     my ($vga) = @_;
3283
3284     my $vgaconf = parse_vga($vga);
3285     my $vgatype = $vgaconf->{type};
3286     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3287
3288     return $1 || 1;
3289 }
3290
3291 sub is_native($) {
3292     my ($arch) = @_;
3293     return get_host_arch() eq $arch;
3294 }
3295
3296 sub get_vm_arch {
3297     my ($conf) = @_;
3298     return $conf->{arch} // get_host_arch();
3299 }
3300
3301 my $default_machines = {
3302     x86_64 => 'pc',
3303     aarch64 => 'virt',
3304 };
3305
3306 sub get_installed_machine_version {
3307     my ($kvmversion) = @_;
3308     $kvmversion = kvm_user_version() if !defined($kvmversion);
3309     $kvmversion =~ m/^(\d+\.\d+)/;
3310     return $1;
3311 }
3312
3313 sub windows_get_pinned_machine_version {
3314     my ($machine, $base_version, $kvmversion) = @_;
3315
3316     my $pin_version = $base_version;
3317     if (!defined($base_version) ||
3318         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3319     ) {
3320         $pin_version = get_installed_machine_version($kvmversion);
3321     }
3322     if (!$machine || $machine eq 'pc') {
3323         $machine = "pc-i440fx-$pin_version";
3324     } elsif ($machine eq 'q35') {
3325         $machine = "pc-q35-$pin_version";
3326     } elsif ($machine eq 'virt') {
3327         $machine = "virt-$pin_version";
3328     } else {
3329         warn "unknown machine type '$machine', not touching that!\n";
3330     }
3331
3332     return $machine;
3333 }
3334
3335 sub get_vm_machine {
3336     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3337
3338     my $machine = $forcemachine || $conf->{machine};
3339
3340     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3341         $kvmversion //= kvm_user_version();
3342         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3343         # layout which confuses windows quite a bit and may result in various regressions..
3344         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3345         if (windows_version($conf->{ostype})) {
3346             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3347         }
3348         $arch //= 'x86_64';
3349         $machine ||= $default_machines->{$arch};
3350         if ($add_pve_version) {
3351             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3352             $machine .= "+pve$pvever";
3353         }
3354     }
3355
3356     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3357         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3358         $machine = $1 if $is_pxe;
3359
3360         # for version-pinned machines that do not include a pve-version (e.g.
3361         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3362         $machine .= '+pve0';
3363
3364         $machine .= '.pxe' if $is_pxe;
3365     }
3366
3367     return $machine;
3368 }
3369
3370 sub get_ovmf_files($$$) {
3371     my ($arch, $efidisk, $smm) = @_;
3372
3373     my $types = $OVMF->{$arch}
3374         or die "no OVMF images known for architecture '$arch'\n";
3375
3376     my $type = 'default';
3377     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3378         $type = $smm ? "4m" : "4m-no-smm";
3379         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3380     }
3381
3382     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3383     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3384     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3385
3386     return ($ovmf_code, $ovmf_vars);
3387 }
3388
3389 my $Arch2Qemu = {
3390     aarch64 => '/usr/bin/qemu-system-aarch64',
3391     x86_64 => '/usr/bin/qemu-system-x86_64',
3392 };
3393 sub get_command_for_arch($) {
3394     my ($arch) = @_;
3395     return '/usr/bin/kvm' if is_native($arch);
3396
3397     my $cmd = $Arch2Qemu->{$arch}
3398         or die "don't know how to emulate architecture '$arch'\n";
3399     return $cmd;
3400 }
3401
3402 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3403 # to use in a QEMU command line (-cpu element), first array_intersect the result
3404 # of query_supported_ with query_understood_. This is necessary because:
3405 #
3406 # a) query_understood_ returns flags the host cannot use and
3407 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3408 #    flags, but CPU settings - with most of them being flags. Those settings
3409 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3410 #
3411 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3412 # expensive. If you need the value returned from this, you can get it much
3413 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3414 # $accel being 'kvm' or 'tcg'.
3415 #
3416 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3417 # changes, automatically populating pmxcfs.
3418 #
3419 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3420 # since kvm and tcg machines support different flags
3421 #
3422 sub query_supported_cpu_flags {
3423     my ($arch) = @_;
3424
3425     $arch //= get_host_arch();
3426     my $default_machine = $default_machines->{$arch};
3427
3428     my $flags = {};
3429
3430     # FIXME: Once this is merged, the code below should work for ARM as well:
3431     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3432     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3433         $arch eq "aarch64";
3434
3435     my $kvm_supported = defined(kvm_version());
3436     my $qemu_cmd = get_command_for_arch($arch);
3437     my $fakevmid = -1;
3438     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3439
3440     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3441     my $query_supported_run_qemu = sub {
3442         my ($kvm) = @_;
3443
3444         my $flags = {};
3445         my $cmd = [
3446             $qemu_cmd,
3447             '-machine', $default_machine,
3448             '-display', 'none',
3449             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3450             '-mon', 'chardev=qmp,mode=control',
3451             '-pidfile', $pidfile,
3452             '-S', '-daemonize'
3453         ];
3454
3455         if (!$kvm) {
3456             push @$cmd, '-accel', 'tcg';
3457         }
3458
3459         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3460         die "QEMU flag querying VM exited with code " . $rc if $rc;
3461
3462         eval {
3463             my $cmd_result = mon_cmd(
3464                 $fakevmid,
3465                 'query-cpu-model-expansion',
3466                 type => 'full',
3467                 model => { name => 'host' }
3468             );
3469
3470             my $props = $cmd_result->{model}->{props};
3471             foreach my $prop (keys %$props) {
3472                 next if $props->{$prop} ne '1';
3473                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3474                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3475                 # We only keep those with underscores, to match /proc/cpuinfo
3476                 $prop =~ s/\.|-/_/g;
3477                 $flags->{$prop} = 1;
3478             }
3479         };
3480         my $err = $@;
3481
3482         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3483         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3484
3485         die $err if $err;
3486
3487         return [ sort keys %$flags ];
3488     };
3489
3490     # We need to query QEMU twice, since KVM and TCG have different supported flags
3491     PVE::QemuConfig->lock_config($fakevmid, sub {
3492         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3493         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3494
3495         if ($kvm_supported) {
3496             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3497             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3498         }
3499     });
3500
3501     return $flags;
3502 }
3503
3504 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3505 my $understood_cpu_flag_dir = "/usr/share/kvm";
3506 sub query_understood_cpu_flags {
3507     my $arch = get_host_arch();
3508     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3509
3510     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3511         if ! -e $filepath;
3512
3513     my $raw = file_get_contents($filepath);
3514     $raw =~ s/^\s+|\s+$//g;
3515     my @flags = split(/\s+/, $raw);
3516
3517     return \@flags;
3518 }
3519
3520 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3521 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3522 my sub should_disable_smm {
3523     my ($conf, $vga) = @_;
3524
3525     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3526         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3527 }
3528
3529 my sub print_ovmf_drive_commandlines {
3530     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3531
3532     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3533
3534     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3535
3536     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3537     if ($d) {
3538         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3539         my ($path, $format) = $d->@{'file', 'format'};
3540         if ($storeid) {
3541             $path = PVE::Storage::path($storecfg, $d->{file});
3542             if (!defined($format)) {
3543                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3544                 $format = qemu_img_format($scfg, $volname);
3545             }
3546         } elsif (!defined($format)) {
3547             die "efidisk format must be specified\n";
3548         }
3549         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3550         if ($path =~ m/^rbd:/) {
3551             $var_drive_str .= ',cache=writeback';
3552             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3553         }
3554         $var_drive_str .= ",format=$format,file=$path";
3555
3556         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3557         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3558     } else {
3559         log_warn("no efidisk configured! Using temporary efivars disk.");
3560         my $path = "/tmp/$vmid-ovmf.fd";
3561         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3562         $var_drive_str .= ",format=raw,file=$path";
3563         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3564     }
3565
3566     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3567 }
3568
3569 sub config_to_command {
3570     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3571         $pbs_backing) = @_;
3572
3573     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3574     my $devices = [];
3575     my $bridges = {};
3576     my $ostype = $conf->{ostype};
3577     my $winversion = windows_version($ostype);
3578     my $kvm = $conf->{kvm};
3579     my $nodename = nodename();
3580
3581     my $arch = get_vm_arch($conf);
3582     my $kvm_binary = get_command_for_arch($arch);
3583     my $kvmver = kvm_user_version($kvm_binary);
3584
3585     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3586         $kvmver //= "undefined";
3587         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3588     }
3589
3590     my $add_pve_version = min_version($kvmver, 4, 1);
3591
3592     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3593     my $machine_version = extract_version($machine_type, $kvmver);
3594     $kvm //= 1 if is_native($arch);
3595
3596     $machine_version =~ m/(\d+)\.(\d+)/;
3597     my ($machine_major, $machine_minor) = ($1, $2);
3598
3599     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3600         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3601     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3602         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3603             ." please upgrade node '$nodename'\n"
3604     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3605         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3606         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3607             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3608             ." node '$nodename'\n";
3609     }
3610
3611     # if a specific +pve version is required for a feature, use $version_guard
3612     # instead of min_version to allow machines to be run with the minimum
3613     # required version
3614     my $required_pve_version = 0;
3615     my $version_guard = sub {
3616         my ($major, $minor, $pve) = @_;
3617         return 0 if !min_version($machine_version, $major, $minor, $pve);
3618         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3619         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3620         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3621         return 1;
3622     };
3623
3624     if ($kvm && !defined kvm_version()) {
3625         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3626             ." or enable in BIOS.\n";
3627     }
3628
3629     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3630     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3631     my $use_old_bios_files = undef;
3632     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3633
3634     my $cmd = [];
3635     if ($conf->{affinity}) {
3636         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3637     }
3638
3639     push @$cmd, $kvm_binary;
3640
3641     push @$cmd, '-id', $vmid;
3642
3643     my $vmname = $conf->{name} || "vm$vmid";
3644
3645     push @$cmd, '-name', "$vmname,debug-threads=on";
3646
3647     push @$cmd, '-no-shutdown';
3648
3649     my $use_virtio = 0;
3650
3651     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3652     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3653     push @$cmd, '-mon', "chardev=qmp,mode=control";
3654
3655     if (min_version($machine_version, 2, 12)) {
3656         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3657         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3658     }
3659
3660     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3661
3662     push @$cmd, '-daemonize';
3663
3664     if ($conf->{smbios1}) {
3665         my $smbios_conf = parse_smbios1($conf->{smbios1});
3666         if ($smbios_conf->{base64}) {
3667             # Do not pass base64 flag to qemu
3668             delete $smbios_conf->{base64};
3669             my $smbios_string = "";
3670             foreach my $key (keys %$smbios_conf) {
3671                 my $value;
3672                 if ($key eq "uuid") {
3673                     $value = $smbios_conf->{uuid}
3674                 } else {
3675                     $value = decode_base64($smbios_conf->{$key});
3676                 }
3677                 # qemu accepts any binary data, only commas need escaping by double comma
3678                 $value =~ s/,/,,/g;
3679                 $smbios_string .= "," . $key . "=" . $value if $value;
3680             }
3681             push @$cmd, '-smbios', "type=1" . $smbios_string;
3682         } else {
3683             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3684         }
3685     }
3686
3687     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3688         my ($code_drive_str, $var_drive_str) =
3689             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3690         push $cmd->@*, '-drive', $code_drive_str;
3691         push $cmd->@*, '-drive', $var_drive_str;
3692     }
3693
3694     if ($q35) { # tell QEMU to load q35 config early
3695         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3696         if (min_version($machine_version, 4, 0)) {
3697             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3698         } else {
3699             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3700         }
3701     }
3702
3703     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3704         push @$cmd, $fixups->@*;
3705     }
3706
3707     if ($conf->{vmgenid}) {
3708         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3709     }
3710
3711     # add usb controllers
3712     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3713         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3714     push @$devices, @usbcontrollers if @usbcontrollers;
3715     my $vga = parse_vga($conf->{vga});
3716
3717     my $qxlnum = vga_conf_has_spice($conf->{vga});
3718     $vga->{type} = 'qxl' if $qxlnum;
3719
3720     if (!$vga->{type}) {
3721         if ($arch eq 'aarch64') {
3722             $vga->{type} = 'virtio';
3723         } elsif (min_version($machine_version, 2, 9)) {
3724             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3725         } else {
3726             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3727         }
3728     }
3729
3730     # enable absolute mouse coordinates (needed by vnc)
3731     my $tablet = $conf->{tablet};
3732     if (!defined($tablet)) {
3733         $tablet = $defaults->{tablet};
3734         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3735         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3736     }
3737
3738     if ($tablet) {
3739         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3740         my $kbd = print_keyboarddevice_full($conf, $arch);
3741         push @$devices, '-device', $kbd if defined($kbd);
3742     }
3743
3744     my $bootorder = device_bootorder($conf);
3745
3746     # host pci device passthrough
3747     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3748         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3749
3750     # usb devices
3751     my $usb_dev_features = {};
3752     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3753
3754     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3755         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3756     push @$devices, @usbdevices if @usbdevices;
3757
3758     # serial devices
3759     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3760         my $path = $conf->{"serial$i"} or next;
3761         if ($path eq 'socket') {
3762             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3763             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3764             # On aarch64, serial0 is the UART device. QEMU only allows
3765             # connecting UART devices via the '-serial' command line, as
3766             # the device has a fixed slot on the hardware...
3767             if ($arch eq 'aarch64' && $i == 0) {
3768                 push @$devices, '-serial', "chardev:serial$i";
3769             } else {
3770                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3771             }
3772         } else {
3773             die "no such serial device\n" if ! -c $path;
3774             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3775             push @$devices, '-device', "isa-serial,chardev=serial$i";
3776         }
3777     }
3778
3779     # parallel devices
3780     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3781         if (my $path = $conf->{"parallel$i"}) {
3782             die "no such parallel device\n" if ! -c $path;
3783             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3784             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3785             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3786         }
3787     }
3788
3789     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3790         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3791         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3792         push @$devices, @$audio_devs;
3793     }
3794
3795     add_tpm_device($vmid, $devices, $conf);
3796
3797     my $sockets = 1;
3798     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3799     $sockets = $conf->{sockets} if  $conf->{sockets};
3800
3801     my $cores = $conf->{cores} || 1;
3802
3803     my $maxcpus = $sockets * $cores;
3804
3805     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3806
3807     my $allowed_vcpus = $cpuinfo->{cpus};
3808
3809     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3810
3811     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3812         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3813         for (my $i = 2; $i <= $vcpus; $i++)  {
3814             my $cpustr = print_cpu_device($conf,$i);
3815             push @$cmd, '-device', $cpustr;
3816         }
3817
3818     } else {
3819
3820         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3821     }
3822     push @$cmd, '-nodefaults';
3823
3824     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3825
3826     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3827
3828     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3829
3830     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3831         push @$devices, '-device', print_vga_device(
3832             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3833
3834         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3835
3836         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3837         push @$cmd,  '-vnc', "unix:$socket,password=on";
3838     } else {
3839         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3840         push @$cmd, '-nographic';
3841     }
3842
3843     # time drift fix
3844     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3845     my $useLocaltime = $conf->{localtime};
3846
3847     if ($winversion >= 5) { # windows
3848         $useLocaltime = 1 if !defined($conf->{localtime});
3849
3850         # use time drift fix when acpi is enabled
3851         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3852             $tdf = 1 if !defined($conf->{tdf});
3853         }
3854     }
3855
3856     if ($winversion >= 6) {
3857         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3858         push @$cmd, '-no-hpet';
3859     }
3860
3861     push @$rtcFlags, 'driftfix=slew' if $tdf;
3862
3863     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3864         push @$rtcFlags, "base=$conf->{startdate}";
3865     } elsif ($useLocaltime) {
3866         push @$rtcFlags, 'base=localtime';
3867     }
3868
3869     if ($forcecpu) {
3870         push @$cmd, '-cpu', $forcecpu;
3871     } else {
3872         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3873     }
3874
3875     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3876
3877     push @$cmd, '-S' if $conf->{freeze};
3878
3879     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3880
3881     my $guest_agent = parse_guest_agent($conf);
3882
3883     if ($guest_agent->{enabled}) {
3884         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3885         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3886
3887         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3888             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3889             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3890             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3891         } elsif ($guest_agent->{type} eq 'isa') {
3892             push @$devices, '-device', "isa-serial,chardev=qga0";
3893         }
3894     }
3895
3896     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3897     if ($rng && $version_guard->(4, 1, 2)) {
3898         check_rng_source($rng->{source});
3899
3900         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3901         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3902         my $limiter_str = "";
3903         if ($max_bytes) {
3904             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3905         }
3906
3907         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3908         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3909         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3910     }
3911
3912     my $spice_port;
3913
3914     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3915         if ($qxlnum > 1) {
3916             if ($winversion){
3917                 for (my $i = 1; $i < $qxlnum; $i++){
3918                     push @$devices, '-device', print_vga_device(
3919                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3920                 }
3921             } else {
3922                 # assume other OS works like Linux
3923                 my ($ram, $vram) = ("134217728", "67108864");
3924                 if ($vga->{memory}) {
3925                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3926                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3927                 }
3928                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3929                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3930             }
3931         }
3932
3933         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3934
3935         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3936         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3937         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3938
3939         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3940         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3941         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3942
3943         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3944         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3945
3946         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3947         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3948         if ($spice_enhancement->{foldersharing}) {
3949             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3950             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3951         }
3952
3953         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3954         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3955             if $spice_enhancement->{videostreaming};
3956
3957         push @$devices, '-spice', "$spice_opts";
3958     }
3959
3960     # enable balloon by default, unless explicitly disabled
3961     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3962         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3963         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3964         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3965         push @$devices, '-device', $ballooncmd;
3966     }
3967
3968     if ($conf->{watchdog}) {
3969         my $wdopts = parse_watchdog($conf->{watchdog});
3970         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3971         my $watchdog = $wdopts->{model} || 'i6300esb';
3972         push @$devices, '-device', "$watchdog$pciaddr";
3973         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3974     }
3975
3976     my $vollist = [];
3977     my $scsicontroller = {};
3978     my $ahcicontroller = {};
3979     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3980
3981     # Add iscsi initiator name if available
3982     if (my $initiator = get_initiator_name()) {
3983         push @$devices, '-iscsi', "initiator-name=$initiator";
3984     }
3985
3986     PVE::QemuConfig->foreach_volume($conf, sub {
3987         my ($ds, $drive) = @_;
3988
3989         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3990             check_volume_storage_type($storecfg, $drive->{file});
3991             push @$vollist, $drive->{file};
3992         }
3993
3994         # ignore efidisk here, already added in bios/fw handling code above
3995         return if $drive->{interface} eq 'efidisk';
3996         # similar for TPM
3997         return if $drive->{interface} eq 'tpmstate';
3998
3999         $use_virtio = 1 if $ds =~ m/^virtio/;
4000
4001         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4002
4003         if ($drive->{interface} eq 'virtio'){
4004            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4005         }
4006
4007         if ($drive->{interface} eq 'scsi') {
4008
4009             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4010
4011             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4012                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4013
4014             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4015             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4016
4017             my $iothread = '';
4018             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4019                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4020                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4021             } elsif ($drive->{iothread}) {
4022                 log_warn(
4023                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4024                 );
4025             }
4026
4027             my $queues = '';
4028             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4029                 $queues = ",num_queues=$drive->{queues}";
4030             }
4031
4032             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4033                 if !$scsicontroller->{$controller};
4034             $scsicontroller->{$controller}=1;
4035         }
4036
4037         if ($drive->{interface} eq 'sata') {
4038             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4039             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4040             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4041                 if !$ahcicontroller->{$controller};
4042             $ahcicontroller->{$controller}=1;
4043         }
4044
4045         my $pbs_conf = $pbs_backing->{$ds};
4046         my $pbs_name = undef;
4047         if ($pbs_conf) {
4048             $pbs_name = "drive-$ds-pbs";
4049             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4050         }
4051
4052         my $drive_cmd = print_drive_commandline_full(
4053             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4054
4055         # extra protection for templates, but SATA and IDE don't support it..
4056         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4057
4058         push @$devices, '-drive',$drive_cmd;
4059         push @$devices, '-device', print_drivedevice_full(
4060             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4061     });
4062
4063     for (my $i = 0; $i < $MAX_NETS; $i++) {
4064         my $netname = "net$i";
4065
4066         next if !$conf->{$netname};
4067         my $d = parse_net($conf->{$netname});
4068         next if !$d;
4069         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4070
4071         $use_virtio = 1 if $d->{model} eq 'virtio';
4072
4073         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4074
4075         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4076         push @$devices, '-netdev', $netdevfull;
4077
4078         my $netdevicefull = print_netdevice_full(
4079             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4080
4081         push @$devices, '-device', $netdevicefull;
4082     }
4083
4084     if ($conf->{ivshmem}) {
4085         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4086
4087         my $bus;
4088         if ($q35) {
4089             $bus = print_pcie_addr("ivshmem");
4090         } else {
4091             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4092         }
4093
4094         my $ivshmem_name = $ivshmem->{name} // $vmid;
4095         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4096
4097         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4098         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4099             .",size=$ivshmem->{size}M";
4100     }
4101
4102     # pci.4 is nested in pci.1
4103     $bridges->{1} = 1 if $bridges->{4};
4104
4105     if (!$q35) { # add pci bridges
4106         if (min_version($machine_version, 2, 3)) {
4107            $bridges->{1} = 1;
4108            $bridges->{2} = 1;
4109         }
4110         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4111     }
4112
4113     for my $k (sort {$b cmp $a} keys %$bridges) {
4114         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4115
4116         my $k_name = $k;
4117         if ($k == 2 && $legacy_igd) {
4118             $k_name = "$k-igd";
4119         }
4120         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4121         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4122
4123         if ($q35) { # add after -readconfig pve-q35.cfg
4124             splice @$devices, 2, 0, '-device', $devstr;
4125         } else {
4126             unshift @$devices, '-device', $devstr if $k > 0;
4127         }
4128     }
4129
4130     if (!$kvm) {
4131         push @$machineFlags, 'accel=tcg';
4132     }
4133
4134     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4135
4136     my $machine_type_min = $machine_type;
4137     if ($add_pve_version) {
4138         $machine_type_min =~ s/\+pve\d+$//;
4139         $machine_type_min .= "+pve$required_pve_version";
4140     }
4141     push @$machineFlags, "type=${machine_type_min}";
4142
4143     push @$cmd, @$devices;
4144     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4145     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4146     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4147
4148     if (my $vmstate = $conf->{vmstate}) {
4149         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4150         push @$vollist, $vmstate;
4151         push @$cmd, '-loadstate', $statepath;
4152         print "activating and using '$vmstate' as vmstate\n";
4153     }
4154
4155     if (PVE::QemuConfig->is_template($conf)) {
4156         # needed to workaround base volumes being read-only
4157         push @$cmd, '-snapshot';
4158     }
4159
4160     # add custom args
4161     if ($conf->{args}) {
4162         my $aa = PVE::Tools::split_args($conf->{args});
4163         push @$cmd, @$aa;
4164     }
4165
4166     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4167 }
4168
4169 sub check_rng_source {
4170     my ($source) = @_;
4171
4172     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4173     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4174         if ! -e $source;
4175
4176     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4177     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4178         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4179         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4180         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4181             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4182             ." to the host.\n";
4183     }
4184 }
4185
4186 sub spice_port {
4187     my ($vmid) = @_;
4188
4189     my $res = mon_cmd($vmid, 'query-spice');
4190
4191     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4192 }
4193
4194 sub vm_devices_list {
4195     my ($vmid) = @_;
4196
4197     my $res = mon_cmd($vmid, 'query-pci');
4198     my $devices_to_check = [];
4199     my $devices = {};
4200     foreach my $pcibus (@$res) {
4201         push @$devices_to_check, @{$pcibus->{devices}},
4202     }
4203
4204     while (@$devices_to_check) {
4205         my $to_check = [];
4206         for my $d (@$devices_to_check) {
4207             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4208             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4209
4210             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4211             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4212         }
4213         $devices_to_check = $to_check;
4214     }
4215
4216     my $resblock = mon_cmd($vmid, 'query-block');
4217     foreach my $block (@$resblock) {
4218         if($block->{device} =~ m/^drive-(\S+)/){
4219                 $devices->{$1} = 1;
4220         }
4221     }
4222
4223     my $resmice = mon_cmd($vmid, 'query-mice');
4224     foreach my $mice (@$resmice) {
4225         if ($mice->{name} eq 'QEMU HID Tablet') {
4226             $devices->{tablet} = 1;
4227             last;
4228         }
4229     }
4230
4231     # for usb devices there is no query-usb
4232     # but we can iterate over the entries in
4233     # qom-list path=/machine/peripheral
4234     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4235     foreach my $per (@$resperipheral) {
4236         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4237             $devices->{$per->{name}} = 1;
4238         }
4239     }
4240
4241     return $devices;
4242 }
4243
4244 sub vm_deviceplug {
4245     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4246
4247     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4248
4249     my $devices_list = vm_devices_list($vmid);
4250     return 1 if defined($devices_list->{$deviceid});
4251
4252     # add PCI bridge if we need it for the device
4253     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4254
4255     if ($deviceid eq 'tablet') {
4256         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4257     } elsif ($deviceid eq 'keyboard') {
4258         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4259     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4260         my $id = $1;
4261         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4262         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4263     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4264         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4265     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4266         qemu_iothread_add($vmid, $deviceid, $device);
4267
4268         qemu_driveadd($storecfg, $vmid, $device);
4269         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4270
4271         qemu_deviceadd($vmid, $devicefull);
4272         eval { qemu_deviceaddverify($vmid, $deviceid); };
4273         if (my $err = $@) {
4274             eval { qemu_drivedel($vmid, $deviceid); };
4275             warn $@ if $@;
4276             die $err;
4277         }
4278     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4279         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4280         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4281         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4282
4283         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4284
4285         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4286             qemu_iothread_add($vmid, $deviceid, $device);
4287             $devicefull .= ",iothread=iothread-$deviceid";
4288         }
4289
4290         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4291             $devicefull .= ",num_queues=$device->{queues}";
4292         }
4293
4294         qemu_deviceadd($vmid, $devicefull);
4295         qemu_deviceaddverify($vmid, $deviceid);
4296     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4297         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4298         qemu_driveadd($storecfg, $vmid, $device);
4299
4300         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4301         eval { qemu_deviceadd($vmid, $devicefull); };
4302         if (my $err = $@) {
4303             eval { qemu_drivedel($vmid, $deviceid); };
4304             warn $@ if $@;
4305             die $err;
4306         }
4307     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4308         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4309
4310         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4311         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4312         my $use_old_bios_files = undef;
4313         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4314
4315         my $netdevicefull = print_netdevice_full(
4316             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4317         qemu_deviceadd($vmid, $netdevicefull);
4318         eval {
4319             qemu_deviceaddverify($vmid, $deviceid);
4320             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4321         };
4322         if (my $err = $@) {
4323             eval { qemu_netdevdel($vmid, $deviceid); };
4324             warn $@ if $@;
4325             die $err;
4326         }
4327     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4328         my $bridgeid = $2;
4329         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4330         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4331
4332         qemu_deviceadd($vmid, $devicefull);
4333         qemu_deviceaddverify($vmid, $deviceid);
4334     } else {
4335         die "can't hotplug device '$deviceid'\n";
4336     }
4337
4338     return 1;
4339 }
4340
4341 # fixme: this should raise exceptions on error!
4342 sub vm_deviceunplug {
4343     my ($vmid, $conf, $deviceid) = @_;
4344
4345     my $devices_list = vm_devices_list($vmid);
4346     return 1 if !defined($devices_list->{$deviceid});
4347
4348     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4349     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4350
4351     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4352         qemu_devicedel($vmid, $deviceid);
4353     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4354         qemu_devicedel($vmid, $deviceid);
4355         qemu_devicedelverify($vmid, $deviceid);
4356     } elsif ($deviceid =~ m/^usb\d+$/) {
4357         qemu_devicedel($vmid, $deviceid);
4358         qemu_devicedelverify($vmid, $deviceid);
4359     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4360         my $device = parse_drive($deviceid, $conf->{$deviceid});
4361
4362         qemu_devicedel($vmid, $deviceid);
4363         qemu_devicedelverify($vmid, $deviceid);
4364         qemu_drivedel($vmid, $deviceid);
4365         qemu_iothread_del($vmid, $deviceid, $device);
4366     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4367         qemu_devicedel($vmid, $deviceid);
4368         qemu_devicedelverify($vmid, $deviceid);
4369     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4370         my $device = parse_drive($deviceid, $conf->{$deviceid});
4371
4372         qemu_devicedel($vmid, $deviceid);
4373         qemu_devicedelverify($vmid, $deviceid);
4374         qemu_drivedel($vmid, $deviceid);
4375         qemu_deletescsihw($conf, $vmid, $deviceid);
4376
4377         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4378             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4379     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4380         qemu_devicedel($vmid, $deviceid);
4381         qemu_devicedelverify($vmid, $deviceid);
4382         qemu_netdevdel($vmid, $deviceid);
4383     } else {
4384         die "can't unplug device '$deviceid'\n";
4385     }
4386
4387     return 1;
4388 }
4389
4390 sub qemu_spice_usbredir_chardev_add {
4391     my ($vmid, $id) = @_;
4392
4393     mon_cmd($vmid, "chardev-add" , (
4394         id => $id,
4395         backend => {
4396             type => 'spicevmc',
4397             data => {
4398                 type => "usbredir",
4399             },
4400         },
4401     ));
4402 }
4403
4404 sub qemu_deviceadd {
4405     my ($vmid, $devicefull) = @_;
4406
4407     $devicefull = "driver=".$devicefull;
4408     my %options =  split(/[=,]/, $devicefull);
4409
4410     mon_cmd($vmid, "device_add" , %options);
4411 }
4412
4413 sub qemu_devicedel {
4414     my ($vmid, $deviceid) = @_;
4415
4416     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4417 }
4418
4419 sub qemu_iothread_add {
4420     my ($vmid, $deviceid, $device) = @_;
4421
4422     if ($device->{iothread}) {
4423         my $iothreads = vm_iothreads_list($vmid);
4424         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4425     }
4426 }
4427
4428 sub qemu_iothread_del {
4429     my ($vmid, $deviceid, $device) = @_;
4430
4431     if ($device->{iothread}) {
4432         my $iothreads = vm_iothreads_list($vmid);
4433         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4434     }
4435 }
4436
4437 sub qemu_objectadd {
4438     my ($vmid, $objectid, $qomtype) = @_;
4439
4440     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4441
4442     return 1;
4443 }
4444
4445 sub qemu_objectdel {
4446     my ($vmid, $objectid) = @_;
4447
4448     mon_cmd($vmid, "object-del", id => $objectid);
4449
4450     return 1;
4451 }
4452
4453 sub qemu_driveadd {
4454     my ($storecfg, $vmid, $device) = @_;
4455
4456     my $kvmver = get_running_qemu_version($vmid);
4457     my $io_uring = min_version($kvmver, 6, 0);
4458     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4459     $drive =~ s/\\/\\\\/g;
4460     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4461
4462     # If the command succeeds qemu prints: "OK"
4463     return 1 if $ret =~ m/OK/s;
4464
4465     die "adding drive failed: $ret\n";
4466 }
4467
4468 sub qemu_drivedel {
4469     my ($vmid, $deviceid) = @_;
4470
4471     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4472     $ret =~ s/^\s+//;
4473
4474     return 1 if $ret eq "";
4475
4476     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4477     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4478
4479     die "deleting drive $deviceid failed : $ret\n";
4480 }
4481
4482 sub qemu_deviceaddverify {
4483     my ($vmid, $deviceid) = @_;
4484
4485     for (my $i = 0; $i <= 5; $i++) {
4486          my $devices_list = vm_devices_list($vmid);
4487          return 1 if defined($devices_list->{$deviceid});
4488          sleep 1;
4489     }
4490
4491     die "error on hotplug device '$deviceid'\n";
4492 }
4493
4494
4495 sub qemu_devicedelverify {
4496     my ($vmid, $deviceid) = @_;
4497
4498     # need to verify that the device is correctly removed as device_del
4499     # is async and empty return is not reliable
4500
4501     for (my $i = 0; $i <= 5; $i++) {
4502          my $devices_list = vm_devices_list($vmid);
4503          return 1 if !defined($devices_list->{$deviceid});
4504          sleep 1;
4505     }
4506
4507     die "error on hot-unplugging device '$deviceid'\n";
4508 }
4509
4510 sub qemu_findorcreatescsihw {
4511     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4512
4513     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4514
4515     my $scsihwid="$controller_prefix$controller";
4516     my $devices_list = vm_devices_list($vmid);
4517
4518     if (!defined($devices_list->{$scsihwid})) {
4519         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4520     }
4521
4522     return 1;
4523 }
4524
4525 sub qemu_deletescsihw {
4526     my ($conf, $vmid, $opt) = @_;
4527
4528     my $device = parse_drive($opt, $conf->{$opt});
4529
4530     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4531         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4532         return 1;
4533     }
4534
4535     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4536
4537     my $devices_list = vm_devices_list($vmid);
4538     foreach my $opt (keys %{$devices_list}) {
4539         if (is_valid_drivename($opt)) {
4540             my $drive = parse_drive($opt, $conf->{$opt});
4541             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4542                 return 1;
4543             }
4544         }
4545     }
4546
4547     my $scsihwid="scsihw$controller";
4548
4549     vm_deviceunplug($vmid, $conf, $scsihwid);
4550
4551     return 1;
4552 }
4553
4554 sub qemu_add_pci_bridge {
4555     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4556
4557     my $bridges = {};
4558
4559     my $bridgeid;
4560
4561     print_pci_addr($device, $bridges, $arch, $machine_type);
4562
4563     while (my ($k, $v) = each %$bridges) {
4564         $bridgeid = $k;
4565     }
4566     return 1 if !defined($bridgeid) || $bridgeid < 1;
4567
4568     my $bridge = "pci.$bridgeid";
4569     my $devices_list = vm_devices_list($vmid);
4570
4571     if (!defined($devices_list->{$bridge})) {
4572         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4573     }
4574
4575     return 1;
4576 }
4577
4578 sub qemu_set_link_status {
4579     my ($vmid, $device, $up) = @_;
4580
4581     mon_cmd($vmid, "set_link", name => $device,
4582                up => $up ? JSON::true : JSON::false);
4583 }
4584
4585 sub qemu_netdevadd {
4586     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4587
4588     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4589     my %options =  split(/[=,]/, $netdev);
4590
4591     if (defined(my $vhost = $options{vhost})) {
4592         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4593     }
4594
4595     if (defined(my $queues = $options{queues})) {
4596         $options{queues} = $queues + 0;
4597     }
4598
4599     mon_cmd($vmid, "netdev_add",  %options);
4600     return 1;
4601 }
4602
4603 sub qemu_netdevdel {
4604     my ($vmid, $deviceid) = @_;
4605
4606     mon_cmd($vmid, "netdev_del", id => $deviceid);
4607 }
4608
4609 sub qemu_usb_hotplug {
4610     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4611
4612     return if !$device;
4613
4614     # remove the old one first
4615     vm_deviceunplug($vmid, $conf, $deviceid);
4616
4617     # check if xhci controller is necessary and available
4618     my $devicelist = vm_devices_list($vmid);
4619
4620     if (!$devicelist->{xhci}) {
4621         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4622         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4623     }
4624
4625     # print_usbdevice_full expects the parsed device
4626     my $d = parse_usb_device($device->{host});
4627     $d->{usb3} = $device->{usb3};
4628
4629     # add the new one
4630     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4631 }
4632
4633 sub qemu_cpu_hotplug {
4634     my ($vmid, $conf, $vcpus) = @_;
4635
4636     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4637
4638     my $sockets = 1;
4639     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4640     $sockets = $conf->{sockets} if  $conf->{sockets};
4641     my $cores = $conf->{cores} || 1;
4642     my $maxcpus = $sockets * $cores;
4643
4644     $vcpus = $maxcpus if !$vcpus;
4645
4646     die "you can't add more vcpus than maxcpus\n"
4647         if $vcpus > $maxcpus;
4648
4649     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4650
4651     if ($vcpus < $currentvcpus) {
4652
4653         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4654
4655             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4656                 qemu_devicedel($vmid, "cpu$i");
4657                 my $retry = 0;
4658                 my $currentrunningvcpus = undef;
4659                 while (1) {
4660                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4661                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4662                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4663                     $retry++;
4664                     sleep 1;
4665                 }
4666                 #update conf after each succesfull cpu unplug
4667                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4668                 PVE::QemuConfig->write_config($vmid, $conf);
4669             }
4670         } else {
4671             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4672         }
4673
4674         return;
4675     }
4676
4677     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4678     die "vcpus in running vm does not match its configuration\n"
4679         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4680
4681     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4682
4683         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4684             my $cpustr = print_cpu_device($conf, $i);
4685             qemu_deviceadd($vmid, $cpustr);
4686
4687             my $retry = 0;
4688             my $currentrunningvcpus = undef;
4689             while (1) {
4690                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4691                 last if scalar(@{$currentrunningvcpus}) == $i;
4692                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4693                 sleep 1;
4694                 $retry++;
4695             }
4696             #update conf after each succesfull cpu hotplug
4697             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4698             PVE::QemuConfig->write_config($vmid, $conf);
4699         }
4700     } else {
4701
4702         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4703             mon_cmd($vmid, "cpu-add", id => int($i));
4704         }
4705     }
4706 }
4707
4708 sub qemu_block_set_io_throttle {
4709     my ($vmid, $deviceid,
4710         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4711         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4712         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4713         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4714
4715     return if !check_running($vmid) ;
4716
4717     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4718         bps => int($bps),
4719         bps_rd => int($bps_rd),
4720         bps_wr => int($bps_wr),
4721         iops => int($iops),
4722         iops_rd => int($iops_rd),
4723         iops_wr => int($iops_wr),
4724         bps_max => int($bps_max),
4725         bps_rd_max => int($bps_rd_max),
4726         bps_wr_max => int($bps_wr_max),
4727         iops_max => int($iops_max),
4728         iops_rd_max => int($iops_rd_max),
4729         iops_wr_max => int($iops_wr_max),
4730         bps_max_length => int($bps_max_length),
4731         bps_rd_max_length => int($bps_rd_max_length),
4732         bps_wr_max_length => int($bps_wr_max_length),
4733         iops_max_length => int($iops_max_length),
4734         iops_rd_max_length => int($iops_rd_max_length),
4735         iops_wr_max_length => int($iops_wr_max_length),
4736     );
4737
4738 }
4739
4740 sub qemu_block_resize {
4741     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4742
4743     my $running = check_running($vmid);
4744
4745     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4746
4747     return if !$running;
4748
4749     my $padding = (1024 - $size % 1024) % 1024;
4750     $size = $size + $padding;
4751
4752     mon_cmd(
4753         $vmid,
4754         "block_resize",
4755         device => $deviceid,
4756         size => int($size),
4757         timeout => 60,
4758     );
4759 }
4760
4761 sub qemu_volume_snapshot {
4762     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4763
4764     my $running = check_running($vmid);
4765
4766     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4767         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4768     } else {
4769         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4770     }
4771 }
4772
4773 sub qemu_volume_snapshot_delete {
4774     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4775
4776     my $running = check_running($vmid);
4777
4778     if($running) {
4779
4780         $running = undef;
4781         my $conf = PVE::QemuConfig->load_config($vmid);
4782         PVE::QemuConfig->foreach_volume($conf, sub {
4783             my ($ds, $drive) = @_;
4784             $running = 1 if $drive->{file} eq $volid;
4785         });
4786     }
4787
4788     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4789         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4790     } else {
4791         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4792     }
4793 }
4794
4795 sub set_migration_caps {
4796     my ($vmid, $savevm) = @_;
4797
4798     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4799
4800     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4801     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4802
4803     my $cap_ref = [];
4804
4805     my $enabled_cap = {
4806         "auto-converge" => 1,
4807         "xbzrle" => 1,
4808         "x-rdma-pin-all" => 0,
4809         "zero-blocks" => 0,
4810         "compress" => 0,
4811         "dirty-bitmaps" => $dirty_bitmaps,
4812     };
4813
4814     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4815
4816     for my $supported_capability (@$supported_capabilities) {
4817         push @$cap_ref, {
4818             capability => $supported_capability->{capability},
4819             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4820         };
4821     }
4822
4823     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4824 }
4825
4826 sub foreach_volid {
4827     my ($conf, $func, @param) = @_;
4828
4829     my $volhash = {};
4830
4831     my $test_volid = sub {
4832         my ($key, $drive, $snapname) = @_;
4833
4834         my $volid = $drive->{file};
4835         return if !$volid;
4836
4837         $volhash->{$volid}->{cdrom} //= 1;
4838         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4839
4840         my $replicate = $drive->{replicate} // 1;
4841         $volhash->{$volid}->{replicate} //= 0;
4842         $volhash->{$volid}->{replicate} = 1 if $replicate;
4843
4844         $volhash->{$volid}->{shared} //= 0;
4845         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4846
4847         $volhash->{$volid}->{referenced_in_config} //= 0;
4848         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4849
4850         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4851             if defined($snapname);
4852
4853         my $size = $drive->{size};
4854         $volhash->{$volid}->{size} //= $size if $size;
4855
4856         $volhash->{$volid}->{is_vmstate} //= 0;
4857         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4858
4859         $volhash->{$volid}->{is_tpmstate} //= 0;
4860         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4861
4862         $volhash->{$volid}->{is_unused} //= 0;
4863         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4864
4865         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4866     };
4867
4868     my $include_opts = {
4869         extra_keys => ['vmstate'],
4870         include_unused => 1,
4871     };
4872
4873     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4874     foreach my $snapname (keys %{$conf->{snapshots}}) {
4875         my $snap = $conf->{snapshots}->{$snapname};
4876         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4877     }
4878
4879     foreach my $volid (keys %$volhash) {
4880         &$func($volid, $volhash->{$volid}, @param);
4881     }
4882 }
4883
4884 my $fast_plug_option = {
4885     'lock' => 1,
4886     'name' => 1,
4887     'onboot' => 1,
4888     'shares' => 1,
4889     'startup' => 1,
4890     'description' => 1,
4891     'protection' => 1,
4892     'vmstatestorage' => 1,
4893     'hookscript' => 1,
4894     'tags' => 1,
4895 };
4896
4897 for my $opt (keys %$confdesc_cloudinit) {
4898     $fast_plug_option->{$opt} = 1;
4899 };
4900
4901 # hotplug changes in [PENDING]
4902 # $selection hash can be used to only apply specified options, for
4903 # example: { cores => 1 } (only apply changed 'cores')
4904 # $errors ref is used to return error messages
4905 sub vmconfig_hotplug_pending {
4906     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4907
4908     my $defaults = load_defaults();
4909     my $arch = get_vm_arch($conf);
4910     my $machine_type = get_vm_machine($conf, undef, $arch);
4911
4912     # commit values which do not have any impact on running VM first
4913     # Note: those option cannot raise errors, we we do not care about
4914     # $selection and always apply them.
4915
4916     my $add_error = sub {
4917         my ($opt, $msg) = @_;
4918         $errors->{$opt} = "hotplug problem - $msg";
4919     };
4920
4921     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4922
4923     my $cloudinit_record_changed = sub {
4924         my ($conf, $opt, $old, $new) = @_;
4925         return if !$cloudinit_pending_properties->{$opt};
4926
4927         my $ci = ($conf->{cloudinit} //= {});
4928
4929         my $recorded = $ci->{$opt};
4930         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4931
4932         if (defined($new)) {
4933             if (defined($old)) {
4934                 # an existing value is being modified
4935                 if (defined($recorded)) {
4936                     # the value was already not in sync
4937                     if ($new eq $recorded) {
4938                         # a value is being reverted to the cloud-init state:
4939                         delete $ci->{$opt};
4940                         delete $added{$opt};
4941                     } else {
4942                         # the value was changed multiple times, do nothing
4943                     }
4944                 } elsif ($added{$opt}) {
4945                     # the value had been marked as added and is being changed, do nothing
4946                 } else {
4947                     # the value is new, record it:
4948                     $ci->{$opt} = $old;
4949                 }
4950             } else {
4951                 # a new value is being added
4952                 if (defined($recorded)) {
4953                     # it was already not in sync
4954                     if ($new eq $recorded) {
4955                         # a value is being reverted to the cloud-init state:
4956                         delete $ci->{$opt};
4957                         delete $added{$opt};
4958                     } else {
4959                         # the value had temporarily been removed, do nothing
4960                     }
4961                 } elsif ($added{$opt}) {
4962                     # the value had been marked as added already, do nothing
4963                 } else {
4964                     # the value is new, add it
4965                     $added{$opt} = 1;
4966                 }
4967             }
4968         } elsif (!defined($old)) {
4969             # a non-existent value is being removed? ignore...
4970         } else {
4971             # a value is being deleted
4972             if (defined($recorded)) {
4973                 # a value was already recorded, just keep it
4974             } elsif ($added{$opt}) {
4975                 # the value was marked as added, remove it
4976                 delete $added{$opt};
4977             } else {
4978                 # a previously unrecorded value is being removed, record the old value:
4979                 $ci->{$opt} = $old;
4980             }
4981         }
4982
4983         my $added = join(',', sort keys %added);
4984         $ci->{added} = $added if length($added);
4985     };
4986
4987     my $changes = 0;
4988     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4989         if ($fast_plug_option->{$opt}) {
4990             my $new = delete $conf->{pending}->{$opt};
4991             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4992             $conf->{$opt} = $new;
4993             $changes = 1;
4994         }
4995     }
4996
4997     if ($changes) {
4998         PVE::QemuConfig->write_config($vmid, $conf);
4999     }
5000
5001     my $ostype = $conf->{ostype};
5002     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5003     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5004     my $usb_hotplug = $hotplug_features->{usb}
5005         && min_version($version, 7, 1)
5006         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5007
5008     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5009     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5010
5011     foreach my $opt (sort keys %$pending_delete_hash) {
5012         next if $selection && !$selection->{$opt};
5013         my $force = $pending_delete_hash->{$opt}->{force};
5014         eval {
5015             if ($opt eq 'hotplug') {
5016                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5017             } elsif ($opt eq 'tablet') {
5018                 die "skip\n" if !$hotplug_features->{usb};
5019                 if ($defaults->{tablet}) {
5020                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5021                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5022                         if $arch eq 'aarch64';
5023                 } else {
5024                     vm_deviceunplug($vmid, $conf, 'tablet');
5025                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5026                 }
5027             } elsif ($opt =~ m/^usb(\d+)$/) {
5028                 my $index = $1;
5029                 die "skip\n" if !$usb_hotplug;
5030                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5031                 vm_deviceunplug($vmid, $conf, $opt);
5032             } elsif ($opt eq 'vcpus') {
5033                 die "skip\n" if !$hotplug_features->{cpu};
5034                 qemu_cpu_hotplug($vmid, $conf, undef);
5035             } elsif ($opt eq 'balloon') {
5036                 # enable balloon device is not hotpluggable
5037                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5038                 # here we reset the ballooning value to memory
5039                 my $balloon = $conf->{memory} || $defaults->{memory};
5040                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5041             } elsif ($fast_plug_option->{$opt}) {
5042                 # do nothing
5043             } elsif ($opt =~ m/^net(\d+)$/) {
5044                 die "skip\n" if !$hotplug_features->{network};
5045                 vm_deviceunplug($vmid, $conf, $opt);
5046             } elsif (is_valid_drivename($opt)) {
5047                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5048                 vm_deviceunplug($vmid, $conf, $opt);
5049                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5050             } elsif ($opt =~ m/^memory$/) {
5051                 die "skip\n" if !$hotplug_features->{memory};
5052                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
5053             } elsif ($opt eq 'cpuunits') {
5054                 $cgroup->change_cpu_shares(undef);
5055             } elsif ($opt eq 'cpulimit') {
5056                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5057             } else {
5058                 die "skip\n";
5059             }
5060         };
5061         if (my $err = $@) {
5062             &$add_error($opt, $err) if $err ne "skip\n";
5063         } else {
5064             my $old = delete $conf->{$opt};
5065             $cloudinit_record_changed->($conf, $opt, $old, undef);
5066             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5067         }
5068     }
5069
5070     my $cloudinit_opt;
5071     foreach my $opt (keys %{$conf->{pending}}) {
5072         next if $selection && !$selection->{$opt};
5073         my $value = $conf->{pending}->{$opt};
5074         eval {
5075             if ($opt eq 'hotplug') {
5076                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5077             } elsif ($opt eq 'tablet') {
5078                 die "skip\n" if !$hotplug_features->{usb};
5079                 if ($value == 1) {
5080                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5081                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5082                         if $arch eq 'aarch64';
5083                 } elsif ($value == 0) {
5084                     vm_deviceunplug($vmid, $conf, 'tablet');
5085                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5086                 }
5087             } elsif ($opt =~ m/^usb(\d+)$/) {
5088                 my $index = $1;
5089                 die "skip\n" if !$usb_hotplug;
5090                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5091                 my $id = $opt;
5092                 if ($d->{host} eq 'spice')  {
5093                     $id = "usbredirdev$index";
5094                 }
5095                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5096             } elsif ($opt eq 'vcpus') {
5097                 die "skip\n" if !$hotplug_features->{cpu};
5098                 qemu_cpu_hotplug($vmid, $conf, $value);
5099             } elsif ($opt eq 'balloon') {
5100                 # enable/disable balloning device is not hotpluggable
5101                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5102                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5103                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5104
5105                 # allow manual ballooning if shares is set to zero
5106                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5107                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5108                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5109                 }
5110             } elsif ($opt =~ m/^net(\d+)$/) {
5111                 # some changes can be done without hotplug
5112                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5113                                     $vmid, $opt, $value, $arch, $machine_type);
5114             } elsif (is_valid_drivename($opt)) {
5115                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5116                 # some changes can be done without hotplug
5117                 my $drive = parse_drive($opt, $value);
5118                 if (drive_is_cloudinit($drive)) {
5119                     $cloudinit_opt = [$opt, $drive];
5120                     # apply all the other changes first, then generate the cloudinit disk
5121                     die "skip\n";
5122                 }
5123                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5124                                      $vmid, $opt, $value, $arch, $machine_type);
5125             } elsif ($opt =~ m/^memory$/) { #dimms
5126                 die "skip\n" if !$hotplug_features->{memory};
5127                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5128             } elsif ($opt eq 'cpuunits') {
5129                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5130                 $cgroup->change_cpu_shares($new_cpuunits);
5131             } elsif ($opt eq 'cpulimit') {
5132                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5133                 $cgroup->change_cpu_quota($cpulimit, 100000);
5134             } elsif ($opt eq 'agent') {
5135                 vmconfig_update_agent($conf, $opt, $value);
5136             } else {
5137                 die "skip\n";  # skip non-hot-pluggable options
5138             }
5139         };
5140         if (my $err = $@) {
5141             &$add_error($opt, $err) if $err ne "skip\n";
5142         } else {
5143             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5144             $conf->{$opt} = $value;
5145             delete $conf->{pending}->{$opt};
5146         }
5147     }
5148
5149     if (defined($cloudinit_opt)) {
5150         my ($opt, $drive) = @$cloudinit_opt;
5151         my $value = $conf->{pending}->{$opt};
5152         eval {
5153             my $temp = {%$conf, $opt => $value};
5154             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5155             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5156                                  $vmid, $opt, $value, $arch, $machine_type);
5157         };
5158         if (my $err = $@) {
5159             &$add_error($opt, $err) if $err ne "skip\n";
5160         } else {
5161             $conf->{$opt} = $value;
5162             delete $conf->{pending}->{$opt};
5163         }
5164     }
5165
5166     # unplug xhci controller if no usb device is left
5167     if ($usb_hotplug) {
5168         my $has_usb = 0;
5169         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5170             next if !defined($conf->{"usb$i"});
5171             $has_usb = 1;
5172             last;
5173         }
5174         if (!$has_usb) {
5175             vm_deviceunplug($vmid, $conf, 'xhci');
5176         }
5177     }
5178
5179     PVE::QemuConfig->write_config($vmid, $conf);
5180
5181     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5182         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5183     }
5184 }
5185
5186 sub try_deallocate_drive {
5187     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5188
5189     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5190         my $volid = $drive->{file};
5191         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5192             my $sid = PVE::Storage::parse_volume_id($volid);
5193             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5194
5195             # check if the disk is really unused
5196             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5197                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5198             PVE::Storage::vdisk_free($storecfg, $volid);
5199             return 1;
5200         } else {
5201             # If vm is not owner of this disk remove from config
5202             return 1;
5203         }
5204     }
5205
5206     return;
5207 }
5208
5209 sub vmconfig_delete_or_detach_drive {
5210     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5211
5212     my $drive = parse_drive($opt, $conf->{$opt});
5213
5214     my $rpcenv = PVE::RPCEnvironment::get();
5215     my $authuser = $rpcenv->get_user();
5216
5217     if ($force) {
5218         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5219         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5220     } else {
5221         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5222     }
5223 }
5224
5225
5226
5227 sub vmconfig_apply_pending {
5228     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5229
5230     return if !scalar(keys %{$conf->{pending}});
5231
5232     my $add_apply_error = sub {
5233         my ($opt, $msg) = @_;
5234         my $err_msg = "unable to apply pending change $opt : $msg";
5235         $errors->{$opt} = $err_msg;
5236         warn $err_msg;
5237     };
5238
5239     # cold plug
5240
5241     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5242     foreach my $opt (sort keys %$pending_delete_hash) {
5243         my $force = $pending_delete_hash->{$opt}->{force};
5244         eval {
5245             if ($opt =~ m/^unused/) {
5246                 die "internal error";
5247             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5248                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5249             }
5250         };
5251         if (my $err = $@) {
5252             $add_apply_error->($opt, $err);
5253         } else {
5254             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5255             delete $conf->{$opt};
5256         }
5257     }
5258
5259     PVE::QemuConfig->cleanup_pending($conf);
5260
5261     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5262
5263     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5264         next if $opt eq 'delete'; # just to be sure
5265         eval {
5266             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5267                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5268             }
5269         };
5270         if (my $err = $@) {
5271             $add_apply_error->($opt, $err);
5272         } else {
5273
5274             if (is_valid_drivename($opt)) {
5275                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5276                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5277             }
5278
5279             $conf->{$opt} = delete $conf->{pending}->{$opt};
5280         }
5281     }
5282
5283     # write all changes at once to avoid unnecessary i/o
5284     PVE::QemuConfig->write_config($vmid, $conf);
5285     if ($generate_cloudinit) {
5286         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5287             # After successful generation and if there were changes to be applied, update the
5288             # config to drop the {cloudinit} entry.
5289             PVE::QemuConfig->write_config($vmid, $conf);
5290         }
5291     }
5292 }
5293
5294 sub vmconfig_update_net {
5295     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5296
5297     my $newnet = parse_net($value);
5298
5299     if ($conf->{$opt}) {
5300         my $oldnet = parse_net($conf->{$opt});
5301
5302         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5303             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5304             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5305             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5306
5307             # for non online change, we try to hot-unplug
5308             die "skip\n" if !$hotplug;
5309             vm_deviceunplug($vmid, $conf, $opt);
5310         } else {
5311
5312             die "internal error" if $opt !~ m/net(\d+)/;
5313             my $iface = "tap${vmid}i$1";
5314
5315             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5316                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5317                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5318                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5319                 PVE::Network::tap_unplug($iface);
5320
5321                 if ($have_sdn) {
5322                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5323                 } else {
5324                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5325                 }
5326             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5327                 # Rate can be applied on its own but any change above needs to
5328                 # include the rate in tap_plug since OVS resets everything.
5329                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5330             }
5331
5332             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5333                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5334             }
5335
5336             return 1;
5337         }
5338     }
5339
5340     if ($hotplug) {
5341         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5342     } else {
5343         die "skip\n";
5344     }
5345 }
5346
5347 sub vmconfig_update_agent {
5348     my ($conf, $opt, $value) = @_;
5349
5350     die "skip\n" if !$conf->{$opt};
5351
5352     my $hotplug_options = { fstrim_cloned_disks => 1 };
5353
5354     my $old_agent = parse_guest_agent($conf);
5355     my $agent = parse_guest_agent({$opt => $value});
5356
5357     for my $option (keys %$agent) { # added/changed options
5358         next if defined($hotplug_options->{$option});
5359         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5360     }
5361
5362     for my $option (keys %$old_agent) { # removed options
5363         next if defined($hotplug_options->{$option});
5364         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5365     }
5366
5367     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5368 }
5369
5370 sub vmconfig_update_disk {
5371     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5372
5373     my $drive = parse_drive($opt, $value);
5374
5375     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5376         my $media = $drive->{media} || 'disk';
5377         my $oldmedia = $old_drive->{media} || 'disk';
5378         die "unable to change media type\n" if $media ne $oldmedia;
5379
5380         if (!drive_is_cdrom($old_drive)) {
5381
5382             if ($drive->{file} ne $old_drive->{file}) {
5383
5384                 die "skip\n" if !$hotplug;
5385
5386                 # unplug and register as unused
5387                 vm_deviceunplug($vmid, $conf, $opt);
5388                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5389
5390             } else {
5391                 # update existing disk
5392
5393                 # skip non hotpluggable value
5394                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5395                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5396                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5397                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5398                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5399                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5400                     die "skip\n";
5401                 }
5402
5403                 # apply throttle
5404                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5405                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5406                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5407                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5408                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5409                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5410                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5411                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5412                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5413                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5414                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5415                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5416                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5417                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5418                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5419                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5420                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5421                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5422
5423                     qemu_block_set_io_throttle(
5424                         $vmid,"drive-$opt",
5425                         ($drive->{mbps} || 0)*1024*1024,
5426                         ($drive->{mbps_rd} || 0)*1024*1024,
5427                         ($drive->{mbps_wr} || 0)*1024*1024,
5428                          $drive->{iops} || 0,
5429                          $drive->{iops_rd} || 0,
5430                          $drive->{iops_wr} || 0,
5431                         ($drive->{mbps_max} || 0)*1024*1024,
5432                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5433                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5434                          $drive->{iops_max} || 0,
5435                          $drive->{iops_rd_max} || 0,
5436                          $drive->{iops_wr_max} || 0,
5437                          $drive->{bps_max_length} || 1,
5438                          $drive->{bps_rd_max_length} || 1,
5439                          $drive->{bps_wr_max_length} || 1,
5440                          $drive->{iops_max_length} || 1,
5441                          $drive->{iops_rd_max_length} || 1,
5442                          $drive->{iops_wr_max_length} || 1,
5443                     );
5444
5445                 }
5446
5447                 return 1;
5448             }
5449
5450         } else { # cdrom
5451
5452             if ($drive->{file} eq 'none') {
5453                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5454                 if (drive_is_cloudinit($old_drive)) {
5455                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5456                 }
5457             } else {
5458                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5459
5460                 # force eject if locked
5461                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5462
5463                 if ($path) {
5464                     mon_cmd($vmid, "blockdev-change-medium",
5465                         id => "$opt", filename => "$path");
5466                 }
5467             }
5468
5469             return 1;
5470         }
5471     }
5472
5473     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5474     # hotplug new disks
5475     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5476     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5477 }
5478
5479 sub vmconfig_update_cloudinit_drive {
5480     my ($storecfg, $conf, $vmid) = @_;
5481
5482     my $cloudinit_ds = undef;
5483     my $cloudinit_drive = undef;
5484
5485     PVE::QemuConfig->foreach_volume($conf, sub {
5486         my ($ds, $drive) = @_;
5487         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5488             $cloudinit_ds = $ds;
5489             $cloudinit_drive = $drive;
5490         }
5491     });
5492
5493     return if !$cloudinit_drive;
5494
5495     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5496         PVE::QemuConfig->write_config($vmid, $conf);
5497     }
5498
5499     my $running = PVE::QemuServer::check_running($vmid);
5500
5501     if ($running) {
5502         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5503         if ($path) {
5504             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5505             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5506         }
5507     }
5508 }
5509
5510 # called in locked context by incoming migration
5511 sub vm_migrate_get_nbd_disks {
5512     my ($storecfg, $conf, $replicated_volumes) = @_;
5513
5514     my $local_volumes = {};
5515     PVE::QemuConfig->foreach_volume($conf, sub {
5516         my ($ds, $drive) = @_;
5517
5518         return if drive_is_cdrom($drive);
5519         return if $ds eq 'tpmstate0';
5520
5521         my $volid = $drive->{file};
5522
5523         return if !$volid;
5524
5525         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5526
5527         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5528         return if $scfg->{shared};
5529
5530         # replicated disks re-use existing state via bitmap
5531         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5532         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5533     });
5534     return $local_volumes;
5535 }
5536
5537 # called in locked context by incoming migration
5538 sub vm_migrate_alloc_nbd_disks {
5539     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5540
5541     my $nbd = {};
5542     foreach my $opt (sort keys %$source_volumes) {
5543         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5544
5545         if ($use_existing) {
5546             $nbd->{$opt}->{drivestr} = print_drive($drive);
5547             $nbd->{$opt}->{volid} = $volid;
5548             $nbd->{$opt}->{replicated} = 1;
5549             next;
5550         }
5551
5552         # storage mapping + volname = regular migration
5553         # storage mapping + format = remote migration
5554         # order of precedence, filtered by whether storage supports it:
5555         # 1. explicit requested format
5556         # 2. format of current volume
5557         # 3. default format of storage
5558         if (!$storagemap->{identity}) {
5559             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5560             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5561             if (!$format || !grep { $format eq $_ } @$validFormats) {
5562                 if ($volname) {
5563                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5564                     my $fileFormat = qemu_img_format($scfg, $volname);
5565                     $format = $fileFormat
5566                         if grep { $fileFormat eq $_ } @$validFormats;
5567                 }
5568                 $format //= $defFormat;
5569             }
5570         } else {
5571             # can't happen for remote migration, so $volname is always defined
5572             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5573             $format = qemu_img_format($scfg, $volname);
5574         }
5575
5576         my $size = $drive->{size} / 1024;
5577         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5578         my $newdrive = $drive;
5579         $newdrive->{format} = $format;
5580         $newdrive->{file} = $newvolid;
5581         my $drivestr = print_drive($newdrive);
5582         $nbd->{$opt}->{drivestr} = $drivestr;
5583         $nbd->{$opt}->{volid} = $newvolid;
5584     }
5585
5586     return $nbd;
5587 }
5588
5589 # see vm_start_nolock for parameters, additionally:
5590 # migrate_opts:
5591 #   storagemap = parsed storage map for allocating NBD disks
5592 sub vm_start {
5593     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5594
5595     return PVE::QemuConfig->lock_config($vmid, sub {
5596         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5597
5598         die "you can't start a vm if it's a template\n"
5599             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5600
5601         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5602         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5603
5604         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5605
5606         if ($has_backup_lock && $running) {
5607             # a backup is currently running, attempt to start the guest in the
5608             # existing QEMU instance
5609             return vm_resume($vmid);
5610         }
5611
5612         PVE::QemuConfig->check_lock($conf)
5613             if !($params->{skiplock} || $has_suspended_lock);
5614
5615         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5616
5617         die "VM $vmid already running\n" if $running;
5618
5619         if (my $storagemap = $migrate_opts->{storagemap}) {
5620             my $replicated = $migrate_opts->{replicated_volumes};
5621             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5622             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5623
5624             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5625                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5626             }
5627         }
5628
5629         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5630     });
5631 }
5632
5633
5634 # params:
5635 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5636 #   skiplock => 0/1, skip checking for config lock
5637 #   skiptemplate => 0/1, skip checking whether VM is template
5638 #   forcemachine => to force QEMU machine (rollback/migration)
5639 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5640 #   timeout => in seconds
5641 #   paused => start VM in paused state (backup)
5642 #   resume => resume from hibernation
5643 #   pbs-backing => {
5644 #      sata0 => {
5645 #         repository
5646 #         snapshot
5647 #         keyfile
5648 #         archive
5649 #      },
5650 #      virtio2 => ...
5651 #   }
5652 # migrate_opts:
5653 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5654 #   migratedfrom => source node
5655 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5656 #   network => CIDR of migration network
5657 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5658 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5659 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5660 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5661 #       contained in config
5662 sub vm_start_nolock {
5663     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5664
5665     my $statefile = $params->{statefile};
5666     my $resume = $params->{resume};
5667
5668     my $migratedfrom = $migrate_opts->{migratedfrom};
5669     my $migration_type = $migrate_opts->{type};
5670
5671     my $res = {};
5672
5673     # clean up leftover reboot request files
5674     eval { clear_reboot_request($vmid); };
5675     warn $@ if $@;
5676
5677     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5678         vmconfig_apply_pending($vmid, $conf, $storecfg);
5679         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5680     }
5681
5682     # don't regenerate the ISO if the VM is started as part of a live migration
5683     # this way we can reuse the old ISO with the correct config
5684     if (!$migratedfrom) {
5685         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5686             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5687             # $conf->{cloudinit}, so we could just not do this?
5688             # But we do it above, so for now let's be consistent.
5689             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5690         }
5691     }
5692
5693     # override offline migrated volumes, conf is out of date still
5694     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5695         for my $key (sort keys $offline_volumes->%*) {
5696             my $parsed = parse_drive($key, $conf->{$key});
5697             $parsed->{file} = $offline_volumes->{$key};
5698             $conf->{$key} = print_drive($parsed);
5699         }
5700     }
5701
5702     my $defaults = load_defaults();
5703
5704     # set environment variable useful inside network script
5705     # for remote migration the config is available on the target node!
5706     if (!$migrate_opts->{remote_node}) {
5707         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5708     }
5709
5710     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5711
5712     my $forcemachine = $params->{forcemachine};
5713     my $forcecpu = $params->{forcecpu};
5714     if ($resume) {
5715         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5716         $forcemachine = $conf->{runningmachine};
5717         $forcecpu = $conf->{runningcpu};
5718         print "Resuming suspended VM\n";
5719     }
5720
5721     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5722         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5723
5724     my $migration_ip;
5725     my $get_migration_ip = sub {
5726         my ($nodename) = @_;
5727
5728         return $migration_ip if defined($migration_ip);
5729
5730         my $cidr = $migrate_opts->{network};
5731
5732         if (!defined($cidr)) {
5733             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5734             $cidr = $dc_conf->{migration}->{network};
5735         }
5736
5737         if (defined($cidr)) {
5738             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5739
5740             die "could not get IP: no address configured on local " .
5741                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5742
5743             die "could not get IP: multiple addresses configured on local " .
5744                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5745
5746             $migration_ip = @$ips[0];
5747         }
5748
5749         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5750             if !defined($migration_ip);
5751
5752         return $migration_ip;
5753     };
5754
5755     if ($statefile) {
5756         if ($statefile eq 'tcp') {
5757             my $migrate = $res->{migrate} = { proto => 'tcp' };
5758             $migrate->{addr} = "localhost";
5759             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5760             my $nodename = nodename();
5761
5762             if (!defined($migration_type)) {
5763                 if (defined($datacenterconf->{migration}->{type})) {
5764                     $migration_type = $datacenterconf->{migration}->{type};
5765                 } else {
5766                     $migration_type = 'secure';
5767                 }
5768             }
5769
5770             if ($migration_type eq 'insecure') {
5771                 $migrate->{addr} = $get_migration_ip->($nodename);
5772                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5773             }
5774
5775             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5776             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5777             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5778             push @$cmd, '-incoming', $migrate->{uri};
5779             push @$cmd, '-S';
5780
5781         } elsif ($statefile eq 'unix') {
5782             # should be default for secure migrations as a ssh TCP forward
5783             # tunnel is not deterministic reliable ready and fails regurarly
5784             # to set up in time, so use UNIX socket forwards
5785             my $migrate = $res->{migrate} = { proto => 'unix' };
5786             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5787             unlink $migrate->{addr};
5788
5789             $migrate->{uri} = "unix:$migrate->{addr}";
5790             push @$cmd, '-incoming', $migrate->{uri};
5791             push @$cmd, '-S';
5792
5793         } elsif (-e $statefile) {
5794             push @$cmd, '-loadstate', $statefile;
5795         } else {
5796             my $statepath = PVE::Storage::path($storecfg, $statefile);
5797             push @$vollist, $statefile;
5798             push @$cmd, '-loadstate', $statepath;
5799         }
5800     } elsif ($params->{paused}) {
5801         push @$cmd, '-S';
5802     }
5803
5804     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5805
5806     my $pci_devices = {}; # host pci devices
5807     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5808         my $dev = $conf->{"hostpci$i"} or next;
5809         $pci_devices->{$i} = parse_hostpci($dev);
5810     }
5811
5812     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5813     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5814
5815     # map to a flat list of pci ids
5816     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5817
5818     # reserve all PCI IDs before actually doing anything with them
5819     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5820
5821     eval {
5822         my $uuid;
5823         for my $id (sort keys %$pci_devices) {
5824             my $d = $pci_devices->{$id};
5825             for my $dev ($d->{pciid}->@*) {
5826                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5827
5828                 # nvidia grid needs the uuid of the mdev as qemu parameter
5829                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5830                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5831                 }
5832             }
5833         }
5834         push @$cmd, '-uuid', $uuid if defined($uuid);
5835     };
5836     if (my $err = $@) {
5837         eval { cleanup_pci_devices($vmid, $conf) };
5838         warn $@ if $@;
5839         die $err;
5840     }
5841
5842     PVE::Storage::activate_volumes($storecfg, $vollist);
5843
5844     eval {
5845         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5846     };
5847     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5848     # timeout should be more than enough here...
5849     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5850
5851     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5852
5853     my %run_params = (
5854         timeout => $statefile ? undef : $start_timeout,
5855         umask => 0077,
5856         noerr => 1,
5857     );
5858
5859     # when migrating, prefix QEMU output so other side can pick up any
5860     # errors that might occur and show the user
5861     if ($migratedfrom) {
5862         $run_params{quiet} = 1;
5863         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5864     }
5865
5866     my %systemd_properties = (
5867         Slice => 'qemu.slice',
5868         KillMode => 'process',
5869         SendSIGKILL => 0,
5870         TimeoutStopUSec => ULONG_MAX, # infinity
5871     );
5872
5873     if (PVE::CGroup::cgroup_mode() == 2) {
5874         $systemd_properties{CPUWeight} = $cpuunits;
5875     } else {
5876         $systemd_properties{CPUShares} = $cpuunits;
5877     }
5878
5879     if (my $cpulimit = $conf->{cpulimit}) {
5880         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5881     }
5882     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5883
5884     my $run_qemu = sub {
5885         PVE::Tools::run_fork sub {
5886             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5887
5888             my $tpmpid;
5889             if (my $tpm = $conf->{tpmstate0}) {
5890                 # start the TPM emulator so QEMU can connect on start
5891                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5892             }
5893
5894             my $exitcode = run_command($cmd, %run_params);
5895             if ($exitcode) {
5896                 if ($tpmpid) {
5897                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5898                     kill 'TERM', $tpmpid;
5899                 }
5900                 die "QEMU exited with code $exitcode\n";
5901             }
5902         };
5903     };
5904
5905     if ($conf->{hugepages}) {
5906
5907         my $code = sub {
5908             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5909             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5910
5911             PVE::QemuServer::Memory::hugepages_mount();
5912             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5913
5914             eval { $run_qemu->() };
5915             if (my $err = $@) {
5916                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5917                     if !$conf->{keephugepages};
5918                 die $err;
5919             }
5920
5921             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5922                 if !$conf->{keephugepages};
5923         };
5924         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5925
5926     } else {
5927         eval { $run_qemu->() };
5928     }
5929
5930     if (my $err = $@) {
5931         # deactivate volumes if start fails
5932         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5933         warn $@ if $@;
5934         eval { cleanup_pci_devices($vmid, $conf) };
5935         warn $@ if $@;
5936
5937         die "start failed: $err";
5938     }
5939
5940     # re-reserve all PCI IDs now that we can know the actual VM PID
5941     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5942     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5943     warn $@ if $@;
5944
5945     if (defined($res->{migrate})) {
5946         print "migration listens on $res->{migrate}->{uri}\n";
5947     } elsif ($statefile) {
5948         eval { mon_cmd($vmid, "cont"); };
5949         warn $@ if $@;
5950     }
5951
5952     #start nbd server for storage migration
5953     if (my $nbd = $migrate_opts->{nbd}) {
5954         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5955
5956         my $migrate_storage_uri;
5957         # nbd_protocol_version > 0 for unix socket support
5958         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5959             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5960             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5961             $migrate_storage_uri = "nbd:unix:$socket_path";
5962             $res->{migrate}->{unix_sockets} = [$socket_path];
5963         } else {
5964             my $nodename = nodename();
5965             my $localip = $get_migration_ip->($nodename);
5966             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5967             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5968
5969             mon_cmd($vmid, "nbd-server-start", addr => {
5970                 type => 'inet',
5971                 data => {
5972                     host => "${localip}",
5973                     port => "${storage_migrate_port}",
5974                 },
5975             });
5976             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5977             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5978         }
5979
5980         my $block_info = mon_cmd($vmid, "query-block");
5981         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5982
5983         foreach my $opt (sort keys %$nbd) {
5984             my $drivestr = $nbd->{$opt}->{drivestr};
5985             my $volid = $nbd->{$opt}->{volid};
5986
5987             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5988
5989             mon_cmd(
5990                 $vmid,
5991                 "block-export-add",
5992                 id => "drive-$opt",
5993                 'node-name' => $block_node,
5994                 writable => JSON::true,
5995                 type => "nbd",
5996                 name => "drive-$opt", # NBD export name
5997             );
5998
5999             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6000             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6001             print "re-using replicated volume: $opt - $volid\n"
6002                 if $nbd->{$opt}->{replicated};
6003
6004             $res->{drives}->{$opt} = $nbd->{$opt};
6005             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6006         }
6007     }
6008
6009     if ($migratedfrom) {
6010         eval {
6011             set_migration_caps($vmid);
6012         };
6013         warn $@ if $@;
6014
6015         if ($spice_port) {
6016             print "spice listens on port $spice_port\n";
6017             $res->{spice_port} = $spice_port;
6018             if ($migrate_opts->{spice_ticket}) {
6019                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6020                     $migrate_opts->{spice_ticket});
6021                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6022             }
6023         }
6024
6025     } else {
6026         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6027             if !$statefile && $conf->{balloon};
6028
6029         foreach my $opt (keys %$conf) {
6030             next if $opt !~  m/^net\d+$/;
6031             my $nicconf = parse_net($conf->{$opt});
6032             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6033         }
6034         add_nets_bridge_fdb($conf, $vmid);
6035     }
6036
6037    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6038         mon_cmd(
6039             $vmid,
6040             'qom-set',
6041             path => "machine/peripheral/balloon0",
6042             property => "guest-stats-polling-interval",
6043             value => 2
6044         );
6045     }
6046
6047     if ($resume) {
6048         print "Resumed VM, removing state\n";
6049         if (my $vmstate = $conf->{vmstate}) {
6050             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6051             PVE::Storage::vdisk_free($storecfg, $vmstate);
6052         }
6053         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6054         PVE::QemuConfig->write_config($vmid, $conf);
6055     }
6056
6057     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6058
6059     return $res;
6060 }
6061
6062 sub vm_commandline {
6063     my ($storecfg, $vmid, $snapname) = @_;
6064
6065     my $conf = PVE::QemuConfig->load_config($vmid);
6066
6067     my ($forcemachine, $forcecpu);
6068     if ($snapname) {
6069         my $snapshot = $conf->{snapshots}->{$snapname};
6070         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6071
6072         # check for machine or CPU overrides in snapshot
6073         $forcemachine = $snapshot->{runningmachine};
6074         $forcecpu = $snapshot->{runningcpu};
6075
6076         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6077
6078         $conf = $snapshot;
6079     }
6080
6081     my $defaults = load_defaults();
6082
6083     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6084
6085     return PVE::Tools::cmd2string($cmd);
6086 }
6087
6088 sub vm_reset {
6089     my ($vmid, $skiplock) = @_;
6090
6091     PVE::QemuConfig->lock_config($vmid, sub {
6092
6093         my $conf = PVE::QemuConfig->load_config($vmid);
6094
6095         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6096
6097         mon_cmd($vmid, "system_reset");
6098     });
6099 }
6100
6101 sub get_vm_volumes {
6102     my ($conf) = @_;
6103
6104     my $vollist = [];
6105     foreach_volid($conf, sub {
6106         my ($volid, $attr) = @_;
6107
6108         return if $volid =~ m|^/|;
6109
6110         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6111         return if !$sid;
6112
6113         push @$vollist, $volid;
6114     });
6115
6116     return $vollist;
6117 }
6118
6119 sub cleanup_pci_devices {
6120     my ($vmid, $conf) = @_;
6121
6122     foreach my $key (keys %$conf) {
6123         next if $key !~ m/^hostpci(\d+)$/;
6124         my $hostpciindex = $1;
6125         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6126         my $d = parse_hostpci($conf->{$key});
6127         if ($d->{mdev}) {
6128             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6129             # don't want to break ABI just for this two liner
6130             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6131             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6132         }
6133     }
6134     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6135 }
6136
6137 sub vm_stop_cleanup {
6138     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6139
6140     eval {
6141
6142         if (!$keepActive) {
6143             my $vollist = get_vm_volumes($conf);
6144             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6145
6146             if (my $tpmdrive = $conf->{tpmstate0}) {
6147                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6148                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6149                 if ($storeid) {
6150                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6151                 }
6152             }
6153         }
6154
6155         foreach my $ext (qw(mon qmp pid vnc qga)) {
6156             unlink "/var/run/qemu-server/${vmid}.$ext";
6157         }
6158
6159         if ($conf->{ivshmem}) {
6160             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6161             # just delete it for now, VMs which have this already open do not
6162             # are affected, but new VMs will get a separated one. If this
6163             # becomes an issue we either add some sort of ref-counting or just
6164             # add a "don't delete on stop" flag to the ivshmem format.
6165             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6166         }
6167
6168         cleanup_pci_devices($vmid, $conf);
6169
6170         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6171     };
6172     warn $@ if $@; # avoid errors - just warn
6173 }
6174
6175 # call only in locked context
6176 sub _do_vm_stop {
6177     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6178
6179     my $pid = check_running($vmid, $nocheck);
6180     return if !$pid;
6181
6182     my $conf;
6183     if (!$nocheck) {
6184         $conf = PVE::QemuConfig->load_config($vmid);
6185         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6186         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6187             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6188             $timeout = $opts->{down} if $opts->{down};
6189         }
6190         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6191     }
6192
6193     eval {
6194         if ($shutdown) {
6195             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6196                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6197             } else {
6198                 mon_cmd($vmid, "system_powerdown");
6199             }
6200         } else {
6201             mon_cmd($vmid, "quit");
6202         }
6203     };
6204     my $err = $@;
6205
6206     if (!$err) {
6207         $timeout = 60 if !defined($timeout);
6208
6209         my $count = 0;
6210         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6211             $count++;
6212             sleep 1;
6213         }
6214
6215         if ($count >= $timeout) {
6216             if ($force) {
6217                 warn "VM still running - terminating now with SIGTERM\n";
6218                 kill 15, $pid;
6219             } else {
6220                 die "VM quit/powerdown failed - got timeout\n";
6221             }
6222         } else {
6223             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6224             return;
6225         }
6226     } else {
6227        if (!check_running($vmid, $nocheck)) {
6228             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6229             return;
6230        }
6231        if ($force) {
6232             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6233             kill 15, $pid;
6234         } else {
6235             die "VM quit/powerdown failed\n";
6236         }
6237     }
6238
6239     # wait again
6240     $timeout = 10;
6241
6242     my $count = 0;
6243     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6244         $count++;
6245         sleep 1;
6246     }
6247
6248     if ($count >= $timeout) {
6249         warn "VM still running - terminating now with SIGKILL\n";
6250         kill 9, $pid;
6251         sleep 1;
6252     }
6253
6254     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6255 }
6256
6257 # Note: use $nocheck to skip tests if VM configuration file exists.
6258 # We need that when migration VMs to other nodes (files already moved)
6259 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6260 sub vm_stop {
6261     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6262
6263     $force = 1 if !defined($force) && !$shutdown;
6264
6265     if ($migratedfrom){
6266         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6267         kill 15, $pid if $pid;
6268         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6269         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6270         return;
6271     }
6272
6273     PVE::QemuConfig->lock_config($vmid, sub {
6274         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6275    });
6276 }
6277
6278 sub vm_reboot {
6279     my ($vmid, $timeout) = @_;
6280
6281     PVE::QemuConfig->lock_config($vmid, sub {
6282         eval {
6283
6284             # only reboot if running, as qmeventd starts it again on a stop event
6285             return if !check_running($vmid);
6286
6287             create_reboot_request($vmid);
6288
6289             my $storecfg = PVE::Storage::config();
6290             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6291
6292         };
6293         if (my $err = $@) {
6294             # avoid that the next normal shutdown will be confused for a reboot
6295             clear_reboot_request($vmid);
6296             die $err;
6297         }
6298    });
6299 }
6300
6301 # note: if using the statestorage parameter, the caller has to check privileges
6302 sub vm_suspend {
6303     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6304
6305     my $conf;
6306     my $path;
6307     my $storecfg;
6308     my $vmstate;
6309
6310     PVE::QemuConfig->lock_config($vmid, sub {
6311
6312         $conf = PVE::QemuConfig->load_config($vmid);
6313
6314         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6315         PVE::QemuConfig->check_lock($conf)
6316             if !($skiplock || $is_backing_up);
6317
6318         die "cannot suspend to disk during backup\n"
6319             if $is_backing_up && $includestate;
6320
6321         if ($includestate) {
6322             $conf->{lock} = 'suspending';
6323             my $date = strftime("%Y-%m-%d", localtime(time()));
6324             $storecfg = PVE::Storage::config();
6325             if (!$statestorage) {
6326                 $statestorage = find_vmstate_storage($conf, $storecfg);
6327                 # check permissions for the storage
6328                 my $rpcenv = PVE::RPCEnvironment::get();
6329                 if ($rpcenv->{type} ne 'cli') {
6330                     my $authuser = $rpcenv->get_user();
6331                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6332                 }
6333             }
6334
6335
6336             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6337                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6338             $path = PVE::Storage::path($storecfg, $vmstate);
6339             PVE::QemuConfig->write_config($vmid, $conf);
6340         } else {
6341             mon_cmd($vmid, "stop");
6342         }
6343     });
6344
6345     if ($includestate) {
6346         # save vm state
6347         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6348
6349         eval {
6350             set_migration_caps($vmid, 1);
6351             mon_cmd($vmid, "savevm-start", statefile => $path);
6352             for(;;) {
6353                 my $state = mon_cmd($vmid, "query-savevm");
6354                 if (!$state->{status}) {
6355                     die "savevm not active\n";
6356                 } elsif ($state->{status} eq 'active') {
6357                     sleep(1);
6358                     next;
6359                 } elsif ($state->{status} eq 'completed') {
6360                     print "State saved, quitting\n";
6361                     last;
6362                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6363                     die "query-savevm failed with error '$state->{error}'\n"
6364                 } else {
6365                     die "query-savevm returned status '$state->{status}'\n";
6366                 }
6367             }
6368         };
6369         my $err = $@;
6370
6371         PVE::QemuConfig->lock_config($vmid, sub {
6372             $conf = PVE::QemuConfig->load_config($vmid);
6373             if ($err) {
6374                 # cleanup, but leave suspending lock, to indicate something went wrong
6375                 eval {
6376                     mon_cmd($vmid, "savevm-end");
6377                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6378                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6379                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6380                     PVE::QemuConfig->write_config($vmid, $conf);
6381                 };
6382                 warn $@ if $@;
6383                 die $err;
6384             }
6385
6386             die "lock changed unexpectedly\n"
6387                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6388
6389             mon_cmd($vmid, "quit");
6390             $conf->{lock} = 'suspended';
6391             PVE::QemuConfig->write_config($vmid, $conf);
6392         });
6393     }
6394 }
6395
6396 # $nocheck is set when called as part of a migration - in this context the
6397 # location of the config file (source or target node) is not deterministic,
6398 # since migration cannot wait for pmxcfs to process the rename
6399 sub vm_resume {
6400     my ($vmid, $skiplock, $nocheck) = @_;
6401
6402     PVE::QemuConfig->lock_config($vmid, sub {
6403         my $res = mon_cmd($vmid, 'query-status');
6404         my $resume_cmd = 'cont';
6405         my $reset = 0;
6406         my $conf;
6407         if ($nocheck) {
6408             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6409             if ($@) {
6410                 my $vmlist = PVE::Cluster::get_vmlist();
6411                 if (exists($vmlist->{ids}->{$vmid})) {
6412                     my $node = $vmlist->{ids}->{$vmid}->{node};
6413                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6414                 }
6415                 if (!$conf) {
6416                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6417                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6418                 }
6419             }
6420         } else {
6421             $conf = PVE::QemuConfig->load_config($vmid);
6422         }
6423
6424         if ($res->{status}) {
6425             return if $res->{status} eq 'running'; # job done, go home
6426             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6427             $reset = 1 if $res->{status} eq 'shutdown';
6428         }
6429
6430         if (!$nocheck) {
6431             PVE::QemuConfig->check_lock($conf)
6432                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6433         }
6434
6435         if ($reset) {
6436             # required if a VM shuts down during a backup and we get a resume
6437             # request before the backup finishes for example
6438             mon_cmd($vmid, "system_reset");
6439         }
6440
6441         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6442
6443         mon_cmd($vmid, $resume_cmd);
6444     });
6445 }
6446
6447 sub vm_sendkey {
6448     my ($vmid, $skiplock, $key) = @_;
6449
6450     PVE::QemuConfig->lock_config($vmid, sub {
6451
6452         my $conf = PVE::QemuConfig->load_config($vmid);
6453
6454         # there is no qmp command, so we use the human monitor command
6455         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6456         die $res if $res ne '';
6457     });
6458 }
6459
6460 # vzdump restore implementaion
6461
6462 sub tar_archive_read_firstfile {
6463     my $archive = shift;
6464
6465     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6466
6467     # try to detect archive type first
6468     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6469         die "unable to open file '$archive'\n";
6470     my $firstfile = <$fh>;
6471     kill 15, $pid;
6472     close $fh;
6473
6474     die "ERROR: archive contaions no data\n" if !$firstfile;
6475     chomp $firstfile;
6476
6477     return $firstfile;
6478 }
6479
6480 sub tar_restore_cleanup {
6481     my ($storecfg, $statfile) = @_;
6482
6483     print STDERR "starting cleanup\n";
6484
6485     if (my $fd = IO::File->new($statfile, "r")) {
6486         while (defined(my $line = <$fd>)) {
6487             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6488                 my $volid = $2;
6489                 eval {
6490                     if ($volid =~ m|^/|) {
6491                         unlink $volid || die 'unlink failed\n';
6492                     } else {
6493                         PVE::Storage::vdisk_free($storecfg, $volid);
6494                     }
6495                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6496                 };
6497                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6498             } else {
6499                 print STDERR "unable to parse line in statfile - $line";
6500             }
6501         }
6502         $fd->close();
6503     }
6504 }
6505
6506 sub restore_file_archive {
6507     my ($archive, $vmid, $user, $opts) = @_;
6508
6509     return restore_vma_archive($archive, $vmid, $user, $opts)
6510         if $archive eq '-';
6511
6512     my $info = PVE::Storage::archive_info($archive);
6513     my $format = $opts->{format} // $info->{format};
6514     my $comp = $info->{compression};
6515
6516     # try to detect archive format
6517     if ($format eq 'tar') {
6518         return restore_tar_archive($archive, $vmid, $user, $opts);
6519     } else {
6520         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6521     }
6522 }
6523
6524 # hepler to remove disks that will not be used after restore
6525 my $restore_cleanup_oldconf = sub {
6526     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6527
6528     my $kept_disks = {};
6529
6530     PVE::QemuConfig->foreach_volume($oldconf, sub {
6531         my ($ds, $drive) = @_;
6532
6533         return if drive_is_cdrom($drive, 1);
6534
6535         my $volid = $drive->{file};
6536         return if !$volid || $volid =~ m|^/|;
6537
6538         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6539         return if !$path || !$owner || ($owner != $vmid);
6540
6541         # Note: only delete disk we want to restore
6542         # other volumes will become unused
6543         if ($virtdev_hash->{$ds}) {
6544             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6545             if (my $err = $@) {
6546                 warn $err;
6547             }
6548         } else {
6549             $kept_disks->{$volid} = 1;
6550         }
6551     });
6552
6553     # after the restore we have no snapshots anymore
6554     for my $snapname (keys $oldconf->{snapshots}->%*) {
6555         my $snap = $oldconf->{snapshots}->{$snapname};
6556         if ($snap->{vmstate}) {
6557             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6558             if (my $err = $@) {
6559                 warn $err;
6560             }
6561         }
6562
6563         for my $volid (keys $kept_disks->%*) {
6564             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6565             warn $@ if $@;
6566         }
6567     }
6568 };
6569
6570 # Helper to parse vzdump backup device hints
6571 #
6572 # $rpcenv: Environment, used to ckeck storage permissions
6573 # $user: User ID, to check storage permissions
6574 # $storecfg: Storage configuration
6575 # $fh: the file handle for reading the configuration
6576 # $devinfo: should contain device sizes for all backu-up'ed devices
6577 # $options: backup options (pool, default storage)
6578 #
6579 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6580 my $parse_backup_hints = sub {
6581     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6582
6583     my $check_storage = sub { # assert if an image can be allocate
6584         my ($storeid, $scfg) = @_;
6585         die "Content type 'images' is not available on storage '$storeid'\n"
6586             if !$scfg->{content}->{images};
6587         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6588             if $user ne 'root@pam';
6589     };
6590
6591     my $virtdev_hash = {};
6592     while (defined(my $line = <$fh>)) {
6593         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6594             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6595             die "archive does not contain data for drive '$virtdev'\n"
6596                 if !$devinfo->{$devname};
6597
6598             if (defined($options->{storage})) {
6599                 $storeid = $options->{storage} || 'local';
6600             } elsif (!$storeid) {
6601                 $storeid = 'local';
6602             }
6603             $format = 'raw' if !$format;
6604             $devinfo->{$devname}->{devname} = $devname;
6605             $devinfo->{$devname}->{virtdev} = $virtdev;
6606             $devinfo->{$devname}->{format} = $format;
6607             $devinfo->{$devname}->{storeid} = $storeid;
6608
6609             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6610             $check_storage->($storeid, $scfg); # permission and content type check
6611
6612             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6613         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6614             my $virtdev = $1;
6615             my $drive = parse_drive($virtdev, $2);
6616
6617             if (drive_is_cloudinit($drive)) {
6618                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6619                 $storeid = $options->{storage} if defined ($options->{storage});
6620                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6621                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6622
6623                 $check_storage->($storeid, $scfg); # permission and content type check
6624
6625                 $virtdev_hash->{$virtdev} = {
6626                     format => $format,
6627                     storeid => $storeid,
6628                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6629                     is_cloudinit => 1,
6630                 };
6631             }
6632         }
6633     }
6634
6635     return $virtdev_hash;
6636 };
6637
6638 # Helper to allocate and activate all volumes required for a restore
6639 #
6640 # $storecfg: Storage configuration
6641 # $virtdev_hash: as returned by parse_backup_hints()
6642 #
6643 # Returns: { $virtdev => $volid }
6644 my $restore_allocate_devices = sub {
6645     my ($storecfg, $virtdev_hash, $vmid) = @_;
6646
6647     my $map = {};
6648     foreach my $virtdev (sort keys %$virtdev_hash) {
6649         my $d = $virtdev_hash->{$virtdev};
6650         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6651         my $storeid = $d->{storeid};
6652         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6653
6654         # test if requested format is supported
6655         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6656         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6657         $d->{format} = $defFormat if !$supported;
6658
6659         my $name;
6660         if ($d->{is_cloudinit}) {
6661             $name = "vm-$vmid-cloudinit";
6662             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6663             if ($scfg->{path}) {
6664                 $name .= ".$d->{format}";
6665             }
6666         }
6667
6668         my $volid = PVE::Storage::vdisk_alloc(
6669             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6670
6671         print STDERR "new volume ID is '$volid'\n";
6672         $d->{volid} = $volid;
6673
6674         PVE::Storage::activate_volumes($storecfg, [$volid]);
6675
6676         $map->{$virtdev} = $volid;
6677     }
6678
6679     return $map;
6680 };
6681
6682 sub restore_update_config_line {
6683     my ($cookie, $map, $line, $unique) = @_;
6684
6685     return '' if $line =~ m/^\#qmdump\#/;
6686     return '' if $line =~ m/^\#vzdump\#/;
6687     return '' if $line =~ m/^lock:/;
6688     return '' if $line =~ m/^unused\d+:/;
6689     return '' if $line =~ m/^parent:/;
6690
6691     my $res = '';
6692
6693     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6694     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6695         # try to convert old 1.X settings
6696         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6697         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6698             my ($model, $macaddr) = split(/\=/, $devconfig);
6699             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6700             my $net = {
6701                 model => $model,
6702                 bridge => "vmbr$ind",
6703                 macaddr => $macaddr,
6704             };
6705             my $netstr = print_net($net);
6706
6707             $res .= "net$cookie->{netcount}: $netstr\n";
6708             $cookie->{netcount}++;
6709         }
6710     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6711         my ($id, $netstr) = ($1, $2);
6712         my $net = parse_net($netstr);
6713         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6714         $netstr = print_net($net);
6715         $res .= "$id: $netstr\n";
6716     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6717         my $virtdev = $1;
6718         my $value = $3;
6719         my $di = parse_drive($virtdev, $value);
6720         if (defined($di->{backup}) && !$di->{backup}) {
6721             $res .= "#$line";
6722         } elsif ($map->{$virtdev}) {
6723             delete $di->{format}; # format can change on restore
6724             $di->{file} = $map->{$virtdev};
6725             $value = print_drive($di);
6726             $res .= "$virtdev: $value\n";
6727         } else {
6728             $res .= $line;
6729         }
6730     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6731         my $vmgenid = $1;
6732         if ($vmgenid ne '0') {
6733             # always generate a new vmgenid if there was a valid one setup
6734             $vmgenid = generate_uuid();
6735         }
6736         $res .= "vmgenid: $vmgenid\n";
6737     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6738         my ($uuid, $uuid_str);
6739         UUID::generate($uuid);
6740         UUID::unparse($uuid, $uuid_str);
6741         my $smbios1 = parse_smbios1($2);
6742         $smbios1->{uuid} = $uuid_str;
6743         $res .= $1.print_smbios1($smbios1)."\n";
6744     } else {
6745         $res .= $line;
6746     }
6747
6748     return $res;
6749 }
6750
6751 my $restore_deactivate_volumes = sub {
6752     my ($storecfg, $virtdev_hash) = @_;
6753
6754     my $vollist = [];
6755     for my $dev (values $virtdev_hash->%*) {
6756         push $vollist->@*, $dev->{volid} if $dev->{volid};
6757     }
6758
6759     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6760     print STDERR $@ if $@;
6761 };
6762
6763 my $restore_destroy_volumes = sub {
6764     my ($storecfg, $virtdev_hash) = @_;
6765
6766     for my $dev (values $virtdev_hash->%*) {
6767         my $volid = $dev->{volid} or next;
6768         eval {
6769             PVE::Storage::vdisk_free($storecfg, $volid);
6770             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6771         };
6772         print STDERR "unable to cleanup '$volid' - $@" if $@;
6773     }
6774 };
6775
6776 my $restore_merge_config = sub {
6777     my ($filename, $backup_conf_raw, $override_conf) = @_;
6778
6779     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6780     for my $key (keys $override_conf->%*) {
6781         $backup_conf->{$key} = $override_conf->{$key};
6782     }
6783
6784     return $backup_conf;
6785 };
6786
6787 sub scan_volids {
6788     my ($cfg, $vmid) = @_;
6789
6790     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6791
6792     my $volid_hash = {};
6793     foreach my $storeid (keys %$info) {
6794         foreach my $item (@{$info->{$storeid}}) {
6795             next if !($item->{volid} && $item->{size});
6796             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6797             $volid_hash->{$item->{volid}} = $item;
6798         }
6799     }
6800
6801     return $volid_hash;
6802 }
6803
6804 sub update_disk_config {
6805     my ($vmid, $conf, $volid_hash) = @_;
6806
6807     my $changes;
6808     my $prefix = "VM $vmid";
6809
6810     # used and unused disks
6811     my $referenced = {};
6812
6813     # Note: it is allowed to define multiple storages with same path (alias), so
6814     # we need to check both 'volid' and real 'path' (two different volid can point
6815     # to the same path).
6816
6817     my $referencedpath = {};
6818
6819     # update size info
6820     PVE::QemuConfig->foreach_volume($conf, sub {
6821         my ($opt, $drive) = @_;
6822
6823         my $volid = $drive->{file};
6824         return if !$volid;
6825         my $volume = $volid_hash->{$volid};
6826
6827         # mark volid as "in-use" for next step
6828         $referenced->{$volid} = 1;
6829         if ($volume && (my $path = $volume->{path})) {
6830             $referencedpath->{$path} = 1;
6831         }
6832
6833         return if drive_is_cdrom($drive);
6834         return if !$volume;
6835
6836         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6837         if (defined($updated)) {
6838             $changes = 1;
6839             $conf->{$opt} = print_drive($updated);
6840             print "$prefix ($opt): $msg\n";
6841         }
6842     });
6843
6844     # remove 'unusedX' entry if volume is used
6845     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6846         my ($opt, $drive) = @_;
6847
6848         my $volid = $drive->{file};
6849         return if !$volid;
6850
6851         my $path;
6852         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6853         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6854             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6855             $changes = 1;
6856             delete $conf->{$opt};
6857         }
6858
6859         $referenced->{$volid} = 1;
6860         $referencedpath->{$path} = 1 if $path;
6861     });
6862
6863     foreach my $volid (sort keys %$volid_hash) {
6864         next if $volid =~ m/vm-$vmid-state-/;
6865         next if $referenced->{$volid};
6866         my $path = $volid_hash->{$volid}->{path};
6867         next if !$path; # just to be sure
6868         next if $referencedpath->{$path};
6869         $changes = 1;
6870         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6871         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6872         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6873     }
6874
6875     return $changes;
6876 }
6877
6878 sub rescan {
6879     my ($vmid, $nolock, $dryrun) = @_;
6880
6881     my $cfg = PVE::Storage::config();
6882
6883     print "rescan volumes...\n";
6884     my $volid_hash = scan_volids($cfg, $vmid);
6885
6886     my $updatefn =  sub {
6887         my ($vmid) = @_;
6888
6889         my $conf = PVE::QemuConfig->load_config($vmid);
6890
6891         PVE::QemuConfig->check_lock($conf);
6892
6893         my $vm_volids = {};
6894         foreach my $volid (keys %$volid_hash) {
6895             my $info = $volid_hash->{$volid};
6896             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6897         }
6898
6899         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6900
6901         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6902     };
6903
6904     if (defined($vmid)) {
6905         if ($nolock) {
6906             &$updatefn($vmid);
6907         } else {
6908             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6909         }
6910     } else {
6911         my $vmlist = config_list();
6912         foreach my $vmid (keys %$vmlist) {
6913             if ($nolock) {
6914                 &$updatefn($vmid);
6915             } else {
6916                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6917             }
6918         }
6919     }
6920 }
6921
6922 sub restore_proxmox_backup_archive {
6923     my ($archive, $vmid, $user, $options) = @_;
6924
6925     my $storecfg = PVE::Storage::config();
6926
6927     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6928     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6929
6930     my $fingerprint = $scfg->{fingerprint};
6931     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6932
6933     my $repo = PVE::PBSClient::get_repository($scfg);
6934     my $namespace = $scfg->{namespace};
6935
6936     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6937     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6938     local $ENV{PBS_PASSWORD} = $password;
6939     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6940
6941     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6942         PVE::Storage::parse_volname($storecfg, $archive);
6943
6944     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6945
6946     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6947
6948     my $tmpdir = "/var/tmp/vzdumptmp$$";
6949     rmtree $tmpdir;
6950     mkpath $tmpdir;
6951
6952     my $conffile = PVE::QemuConfig->config_file($vmid);
6953      # disable interrupts (always do cleanups)
6954     local $SIG{INT} =
6955         local $SIG{TERM} =
6956         local $SIG{QUIT} =
6957         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6958
6959     # Note: $oldconf is undef if VM does not exists
6960     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6961     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6962     my $new_conf_raw = '';
6963
6964     my $rpcenv = PVE::RPCEnvironment::get();
6965     my $devinfo = {}; # info about drives included in backup
6966     my $virtdev_hash = {}; # info about allocated drives
6967
6968     eval {
6969         # enable interrupts
6970         local $SIG{INT} =
6971             local $SIG{TERM} =
6972             local $SIG{QUIT} =
6973             local $SIG{HUP} =
6974             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6975
6976         my $cfgfn = "$tmpdir/qemu-server.conf";
6977         my $firewall_config_fn = "$tmpdir/fw.conf";
6978         my $index_fn = "$tmpdir/index.json";
6979
6980         my $cmd = "restore";
6981
6982         my $param = [$pbs_backup_name, "index.json", $index_fn];
6983         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6984         my $index = PVE::Tools::file_get_contents($index_fn);
6985         $index = decode_json($index);
6986
6987         foreach my $info (@{$index->{files}}) {
6988             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6989                 my $devname = $1;
6990                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6991                     $devinfo->{$devname}->{size} = $1;
6992                 } else {
6993                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6994                 }
6995             }
6996         }
6997
6998         my $is_qemu_server_backup = scalar(
6999             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7000         );
7001         if (!$is_qemu_server_backup) {
7002             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7003         }
7004         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7005
7006         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7007         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7008
7009         if ($has_firewall_config) {
7010             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7011             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7012
7013             my $pve_firewall_dir = '/etc/pve/firewall';
7014             mkdir $pve_firewall_dir; # make sure the dir exists
7015             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7016         }
7017
7018         my $fh = IO::File->new($cfgfn, "r") ||
7019             die "unable to read qemu-server.conf - $!\n";
7020
7021         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7022
7023         # fixme: rate limit?
7024
7025         # create empty/temp config
7026         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7027
7028         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7029
7030         # allocate volumes
7031         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7032
7033         foreach my $virtdev (sort keys %$virtdev_hash) {
7034             my $d = $virtdev_hash->{$virtdev};
7035             next if $d->{is_cloudinit}; # no need to restore cloudinit
7036
7037             # this fails if storage is unavailable
7038             my $volid = $d->{volid};
7039             my $path = PVE::Storage::path($storecfg, $volid);
7040
7041             # for live-restore we only want to preload the efidisk and TPM state
7042             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7043
7044             my @ns_arg;
7045             if (defined(my $ns = $scfg->{namespace})) {
7046                 @ns_arg = ('--ns', $ns);
7047             }
7048
7049             my $pbs_restore_cmd = [
7050                 '/usr/bin/pbs-restore',
7051                 '--repository', $repo,
7052                 @ns_arg,
7053                 $pbs_backup_name,
7054                 "$d->{devname}.img.fidx",
7055                 $path,
7056                 '--verbose',
7057                 ];
7058
7059             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7060             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7061
7062             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7063                 push @$pbs_restore_cmd, '--skip-zero';
7064             }
7065
7066             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7067             print "restore proxmox backup image: $dbg_cmdstring\n";
7068             run_command($pbs_restore_cmd);
7069         }
7070
7071         $fh->seek(0, 0) || die "seek failed - $!\n";
7072
7073         my $cookie = { netcount => 0 };
7074         while (defined(my $line = <$fh>)) {
7075             $new_conf_raw .= restore_update_config_line(
7076                 $cookie,
7077                 $map,
7078                 $line,
7079                 $options->{unique},
7080             );
7081         }
7082
7083         $fh->close();
7084     };
7085     my $err = $@;
7086
7087     if ($err || !$options->{live}) {
7088         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7089     }
7090
7091     rmtree $tmpdir;
7092
7093     if ($err) {
7094         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7095         die $err;
7096     }
7097
7098     if ($options->{live}) {
7099         # keep lock during live-restore
7100         $new_conf_raw .= "\nlock: create";
7101     }
7102
7103     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7104     PVE::QemuConfig->write_config($vmid, $new_conf);
7105
7106     eval { rescan($vmid, 1); };
7107     warn $@ if $@;
7108
7109     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7110
7111     if ($options->{live}) {
7112         # enable interrupts
7113         local $SIG{INT} =
7114             local $SIG{TERM} =
7115             local $SIG{QUIT} =
7116             local $SIG{HUP} =
7117             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7118
7119         my $conf = PVE::QemuConfig->load_config($vmid);
7120         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7121
7122         # these special drives are already restored before start
7123         delete $devinfo->{'drive-efidisk0'};
7124         delete $devinfo->{'drive-tpmstate0-backup'};
7125
7126         my $pbs_opts = {
7127             repo => $repo,
7128             keyfile => $keyfile,
7129             snapshot => $pbs_backup_name,
7130             namespace => $namespace,
7131         };
7132         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7133
7134         PVE::QemuConfig->remove_lock($vmid, "create");
7135     }
7136 }
7137
7138 sub pbs_live_restore {
7139     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7140
7141     print "starting VM for live-restore\n";
7142     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7143
7144     my $pbs_backing = {};
7145     for my $ds (keys %$restored_disks) {
7146         $ds =~ m/^drive-(.*)$/;
7147         my $confname = $1;
7148         $pbs_backing->{$confname} = {
7149             repository => $opts->{repo},
7150             snapshot => $opts->{snapshot},
7151             archive => "$ds.img.fidx",
7152         };
7153         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7154         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7155
7156         my $drive = parse_drive($confname, $conf->{$confname});
7157         print "restoring '$ds' to '$drive->{file}'\n";
7158     }
7159
7160     my $drives_streamed = 0;
7161     eval {
7162         # make sure HA doesn't interrupt our restore by stopping the VM
7163         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7164             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7165         }
7166
7167         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7168         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7169         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7170
7171         my $qmeventd_fd = register_qmeventd_handle($vmid);
7172
7173         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7174         # this will effectively collapse the backing image chain consisting of
7175         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7176         # removes itself once all backing images vanish with 'auto-remove=on')
7177         my $jobs = {};
7178         for my $ds (sort keys %$restored_disks) {
7179             my $job_id = "restore-$ds";
7180             mon_cmd($vmid, 'block-stream',
7181                 'job-id' => $job_id,
7182                 device => "$ds",
7183             );
7184             $jobs->{$job_id} = {};
7185         }
7186
7187         mon_cmd($vmid, 'cont');
7188         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7189
7190         print "restore-drive jobs finished successfully, removing all tracking block devices"
7191             ." to disconnect from Proxmox Backup Server\n";
7192
7193         for my $ds (sort keys %$restored_disks) {
7194             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7195         }
7196
7197         close($qmeventd_fd);
7198     };
7199
7200     my $err = $@;
7201
7202     if ($err) {
7203         warn "An error occurred during live-restore: $err\n";
7204         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7205         die "live-restore failed\n";
7206     }
7207 }
7208
7209 sub restore_vma_archive {
7210     my ($archive, $vmid, $user, $opts, $comp) = @_;
7211
7212     my $readfrom = $archive;
7213
7214     my $cfg = PVE::Storage::config();
7215     my $commands = [];
7216     my $bwlimit = $opts->{bwlimit};
7217
7218     my $dbg_cmdstring = '';
7219     my $add_pipe = sub {
7220         my ($cmd) = @_;
7221         push @$commands, $cmd;
7222         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7223         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7224         $readfrom = '-';
7225     };
7226
7227     my $input = undef;
7228     if ($archive eq '-') {
7229         $input = '<&STDIN';
7230     } else {
7231         # If we use a backup from a PVE defined storage we also consider that
7232         # storage's rate limit:
7233         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7234         if (defined($volid)) {
7235             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7236             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7237             if ($readlimit) {
7238                 print STDERR "applying read rate limit: $readlimit\n";
7239                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7240                 $add_pipe->($cstream);
7241             }
7242         }
7243     }
7244
7245     if ($comp) {
7246         my $info = PVE::Storage::decompressor_info('vma', $comp);
7247         my $cmd = $info->{decompressor};
7248         push @$cmd, $readfrom;
7249         $add_pipe->($cmd);
7250     }
7251
7252     my $tmpdir = "/var/tmp/vzdumptmp$$";
7253     rmtree $tmpdir;
7254
7255     # disable interrupts (always do cleanups)
7256     local $SIG{INT} =
7257         local $SIG{TERM} =
7258         local $SIG{QUIT} =
7259         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7260
7261     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7262     POSIX::mkfifo($mapfifo, 0600);
7263     my $fifofh;
7264     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7265
7266     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7267
7268     my $oldtimeout;
7269     my $timeout = 5;
7270
7271     my $devinfo = {}; # info about drives included in backup
7272     my $virtdev_hash = {}; # info about allocated drives
7273
7274     my $rpcenv = PVE::RPCEnvironment::get();
7275
7276     my $conffile = PVE::QemuConfig->config_file($vmid);
7277
7278     # Note: $oldconf is undef if VM does not exist
7279     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7280     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7281     my $new_conf_raw = '';
7282
7283     my %storage_limits;
7284
7285     my $print_devmap = sub {
7286         my $cfgfn = "$tmpdir/qemu-server.conf";
7287
7288         # we can read the config - that is already extracted
7289         my $fh = IO::File->new($cfgfn, "r") ||
7290             die "unable to read qemu-server.conf - $!\n";
7291
7292         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7293         if (-f $fwcfgfn) {
7294             my $pve_firewall_dir = '/etc/pve/firewall';
7295             mkdir $pve_firewall_dir; # make sure the dir exists
7296             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7297         }
7298
7299         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7300
7301         foreach my $info (values %{$virtdev_hash}) {
7302             my $storeid = $info->{storeid};
7303             next if defined($storage_limits{$storeid});
7304
7305             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7306             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7307             $storage_limits{$storeid} = $limit * 1024;
7308         }
7309
7310         foreach my $devname (keys %$devinfo) {
7311             die "found no device mapping information for device '$devname'\n"
7312                 if !$devinfo->{$devname}->{virtdev};
7313         }
7314
7315         # create empty/temp config
7316         if ($oldconf) {
7317             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7318             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7319         }
7320
7321         # allocate volumes
7322         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7323
7324         # print restore information to $fifofh
7325         foreach my $virtdev (sort keys %$virtdev_hash) {
7326             my $d = $virtdev_hash->{$virtdev};
7327             next if $d->{is_cloudinit}; # no need to restore cloudinit
7328
7329             my $storeid = $d->{storeid};
7330             my $volid = $d->{volid};
7331
7332             my $map_opts = '';
7333             if (my $limit = $storage_limits{$storeid}) {
7334                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7335             }
7336
7337             my $write_zeros = 1;
7338             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7339                 $write_zeros = 0;
7340             }
7341
7342             my $path = PVE::Storage::path($cfg, $volid);
7343
7344             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7345
7346             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7347         }
7348
7349         $fh->seek(0, 0) || die "seek failed - $!\n";
7350
7351         my $cookie = { netcount => 0 };
7352         while (defined(my $line = <$fh>)) {
7353             $new_conf_raw .= restore_update_config_line(
7354                 $cookie,
7355                 $map,
7356                 $line,
7357                 $opts->{unique},
7358             );
7359         }
7360
7361         $fh->close();
7362     };
7363
7364     eval {
7365         # enable interrupts
7366         local $SIG{INT} =
7367             local $SIG{TERM} =
7368             local $SIG{QUIT} =
7369             local $SIG{HUP} =
7370             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7371         local $SIG{ALRM} = sub { die "got timeout\n"; };
7372
7373         $oldtimeout = alarm($timeout);
7374
7375         my $parser = sub {
7376             my $line = shift;
7377
7378             print "$line\n";
7379
7380             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7381                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7382                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7383             } elsif ($line =~ m/^CTIME: /) {
7384                 # we correctly received the vma config, so we can disable
7385                 # the timeout now for disk allocation (set to 10 minutes, so
7386                 # that we always timeout if something goes wrong)
7387                 alarm(600);
7388                 &$print_devmap();
7389                 print $fifofh "done\n";
7390                 my $tmp = $oldtimeout || 0;
7391                 $oldtimeout = undef;
7392                 alarm($tmp);
7393                 close($fifofh);
7394                 $fifofh = undef;
7395             }
7396         };
7397
7398         print "restore vma archive: $dbg_cmdstring\n";
7399         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7400     };
7401     my $err = $@;
7402
7403     alarm($oldtimeout) if $oldtimeout;
7404
7405     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7406
7407     close($fifofh) if $fifofh;
7408     unlink $mapfifo;
7409     rmtree $tmpdir;
7410
7411     if ($err) {
7412         $restore_destroy_volumes->($cfg, $virtdev_hash);
7413         die $err;
7414     }
7415
7416     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7417     PVE::QemuConfig->write_config($vmid, $new_conf);
7418
7419     eval { rescan($vmid, 1); };
7420     warn $@ if $@;
7421
7422     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7423 }
7424
7425 sub restore_tar_archive {
7426     my ($archive, $vmid, $user, $opts) = @_;
7427
7428     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7429         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7430         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7431     }
7432
7433     if ($archive ne '-') {
7434         my $firstfile = tar_archive_read_firstfile($archive);
7435         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7436             if $firstfile ne 'qemu-server.conf';
7437     }
7438
7439     my $storecfg = PVE::Storage::config();
7440
7441     # avoid zombie disks when restoring over an existing VM -> cleanup first
7442     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7443     # skiplock=1 because qmrestore has set the 'create' lock itself already
7444     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7445     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7446
7447     my $tocmd = "/usr/lib/qemu-server/qmextract";
7448
7449     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7450     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7451     $tocmd .= ' --prealloc' if $opts->{prealloc};
7452     $tocmd .= ' --info' if $opts->{info};
7453
7454     # tar option "xf" does not autodetect compression when read from STDIN,
7455     # so we pipe to zcat
7456     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7457         PVE::Tools::shellquote("--to-command=$tocmd");
7458
7459     my $tmpdir = "/var/tmp/vzdumptmp$$";
7460     mkpath $tmpdir;
7461
7462     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7463     local $ENV{VZDUMP_VMID} = $vmid;
7464     local $ENV{VZDUMP_USER} = $user;
7465
7466     my $conffile = PVE::QemuConfig->config_file($vmid);
7467     my $new_conf_raw = '';
7468
7469     # disable interrupts (always do cleanups)
7470     local $SIG{INT} =
7471         local $SIG{TERM} =
7472         local $SIG{QUIT} =
7473         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7474
7475     eval {
7476         # enable interrupts
7477         local $SIG{INT} =
7478             local $SIG{TERM} =
7479             local $SIG{QUIT} =
7480             local $SIG{HUP} =
7481             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7482
7483         if ($archive eq '-') {
7484             print "extracting archive from STDIN\n";
7485             run_command($cmd, input => "<&STDIN");
7486         } else {
7487             print "extracting archive '$archive'\n";
7488             run_command($cmd);
7489         }
7490
7491         return if $opts->{info};
7492
7493         # read new mapping
7494         my $map = {};
7495         my $statfile = "$tmpdir/qmrestore.stat";
7496         if (my $fd = IO::File->new($statfile, "r")) {
7497             while (defined (my $line = <$fd>)) {
7498                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7499                     $map->{$1} = $2 if $1;
7500                 } else {
7501                     print STDERR "unable to parse line in statfile - $line\n";
7502                 }
7503             }
7504             $fd->close();
7505         }
7506
7507         my $confsrc = "$tmpdir/qemu-server.conf";
7508
7509         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7510
7511         my $cookie = { netcount => 0 };
7512         while (defined (my $line = <$srcfd>)) {
7513             $new_conf_raw .= restore_update_config_line(
7514                 $cookie,
7515                 $map,
7516                 $line,
7517                 $opts->{unique},
7518             );
7519         }
7520
7521         $srcfd->close();
7522     };
7523     if (my $err = $@) {
7524         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7525         die $err;
7526     }
7527
7528     rmtree $tmpdir;
7529
7530     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7531
7532     PVE::Cluster::cfs_update(); # make sure we read new file
7533
7534     eval { rescan($vmid, 1); };
7535     warn $@ if $@;
7536 };
7537
7538 sub foreach_storage_used_by_vm {
7539     my ($conf, $func) = @_;
7540
7541     my $sidhash = {};
7542
7543     PVE::QemuConfig->foreach_volume($conf, sub {
7544         my ($ds, $drive) = @_;
7545         return if drive_is_cdrom($drive);
7546
7547         my $volid = $drive->{file};
7548
7549         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7550         $sidhash->{$sid} = $sid if $sid;
7551     });
7552
7553     foreach my $sid (sort keys %$sidhash) {
7554         &$func($sid);
7555     }
7556 }
7557
7558 my $qemu_snap_storage = {
7559     rbd => 1,
7560 };
7561 sub do_snapshots_with_qemu {
7562     my ($storecfg, $volid, $deviceid) = @_;
7563
7564     return if $deviceid =~ m/tpmstate0/;
7565
7566     my $storage_name = PVE::Storage::parse_volume_id($volid);
7567     my $scfg = $storecfg->{ids}->{$storage_name};
7568     die "could not find storage '$storage_name'\n" if !defined($scfg);
7569
7570     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7571         return 1;
7572     }
7573
7574     if ($volid =~ m/\.(qcow2|qed)$/){
7575         return 1;
7576     }
7577
7578     return;
7579 }
7580
7581 sub qga_check_running {
7582     my ($vmid, $nowarn) = @_;
7583
7584     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7585     if ($@) {
7586         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7587         return 0;
7588     }
7589     return 1;
7590 }
7591
7592 sub template_create {
7593     my ($vmid, $conf, $disk) = @_;
7594
7595     my $storecfg = PVE::Storage::config();
7596
7597     PVE::QemuConfig->foreach_volume($conf, sub {
7598         my ($ds, $drive) = @_;
7599
7600         return if drive_is_cdrom($drive);
7601         return if $disk && $ds ne $disk;
7602
7603         my $volid = $drive->{file};
7604         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7605
7606         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7607         $drive->{file} = $voliddst;
7608         $conf->{$ds} = print_drive($drive);
7609         PVE::QemuConfig->write_config($vmid, $conf);
7610     });
7611 }
7612
7613 sub convert_iscsi_path {
7614     my ($path) = @_;
7615
7616     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7617         my $portal = $1;
7618         my $target = $2;
7619         my $lun = $3;
7620
7621         my $initiator_name = get_initiator_name();
7622
7623         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7624                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7625     }
7626
7627     die "cannot convert iscsi path '$path', unkown format\n";
7628 }
7629
7630 sub qemu_img_convert {
7631     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7632
7633     my $storecfg = PVE::Storage::config();
7634     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7635     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7636
7637     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7638
7639     my $cachemode;
7640     my $src_path;
7641     my $src_is_iscsi = 0;
7642     my $src_format;
7643
7644     if ($src_storeid) {
7645         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7646         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7647         $src_format = qemu_img_format($src_scfg, $src_volname);
7648         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7649         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7650         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7651     } elsif (-f $src_volid || -b $src_volid) {
7652         $src_path = $src_volid;
7653         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7654             $src_format = $1;
7655         }
7656     }
7657
7658     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7659
7660     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7661     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7662     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7663     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7664
7665     my $cmd = [];
7666     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7667     push @$cmd, '-l', "snapshot.name=$snapname"
7668         if $snapname && $src_format && $src_format eq "qcow2";
7669     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7670     push @$cmd, '-T', $cachemode if defined($cachemode);
7671
7672     if ($src_is_iscsi) {
7673         push @$cmd, '--image-opts';
7674         $src_path = convert_iscsi_path($src_path);
7675     } elsif ($src_format) {
7676         push @$cmd, '-f', $src_format;
7677     }
7678
7679     if ($dst_is_iscsi) {
7680         push @$cmd, '--target-image-opts';
7681         $dst_path = convert_iscsi_path($dst_path);
7682     } else {
7683         push @$cmd, '-O', $dst_format;
7684     }
7685
7686     push @$cmd, $src_path;
7687
7688     if (!$dst_is_iscsi && $is_zero_initialized) {
7689         push @$cmd, "zeroinit:$dst_path";
7690     } else {
7691         push @$cmd, $dst_path;
7692     }
7693
7694     my $parser = sub {
7695         my $line = shift;
7696         if($line =~ m/\((\S+)\/100\%\)/){
7697             my $percent = $1;
7698             my $transferred = int($size * $percent / 100);
7699             my $total_h = render_bytes($size, 1);
7700             my $transferred_h = render_bytes($transferred, 1);
7701
7702             print "transferred $transferred_h of $total_h ($percent%)\n";
7703         }
7704
7705     };
7706
7707     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7708     my $err = $@;
7709     die "copy failed: $err" if $err;
7710 }
7711
7712 sub qemu_img_format {
7713     my ($scfg, $volname) = @_;
7714
7715     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7716         return $1;
7717     } else {
7718         return "raw";
7719     }
7720 }
7721
7722 sub qemu_drive_mirror {
7723     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7724
7725     $jobs = {} if !$jobs;
7726
7727     my $qemu_target;
7728     my $format;
7729     $jobs->{"drive-$drive"} = {};
7730
7731     if ($dst_volid =~ /^nbd:/) {
7732         $qemu_target = $dst_volid;
7733         $format = "nbd";
7734     } else {
7735         my $storecfg = PVE::Storage::config();
7736         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7737
7738         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7739
7740         $format = qemu_img_format($dst_scfg, $dst_volname);
7741
7742         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7743
7744         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7745     }
7746
7747     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7748     $opts->{format} = $format if $format;
7749
7750     if (defined($src_bitmap)) {
7751         $opts->{sync} = 'incremental';
7752         $opts->{bitmap} = $src_bitmap;
7753         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7754     }
7755
7756     if (defined($bwlimit)) {
7757         $opts->{speed} = $bwlimit * 1024;
7758         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7759     } else {
7760         print "drive mirror is starting for drive-$drive\n";
7761     }
7762
7763     # if a job already runs for this device we get an error, catch it for cleanup
7764     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7765     if (my $err = $@) {
7766         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7767         warn "$@\n" if $@;
7768         die "mirroring error: $err\n";
7769     }
7770
7771     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7772 }
7773
7774 # $completion can be either
7775 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7776 # 'cancel': wait until all jobs are ready, block-job-cancel them
7777 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7778 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7779 sub qemu_drive_mirror_monitor {
7780     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7781
7782     $completion //= 'complete';
7783     $op //= "mirror";
7784
7785     eval {
7786         my $err_complete = 0;
7787
7788         my $starttime = time ();
7789         while (1) {
7790             die "block job ('$op') timed out\n" if $err_complete > 300;
7791
7792             my $stats = mon_cmd($vmid, "query-block-jobs");
7793             my $ctime = time();
7794
7795             my $running_jobs = {};
7796             for my $stat (@$stats) {
7797                 next if $stat->{type} ne $op;
7798                 $running_jobs->{$stat->{device}} = $stat;
7799             }
7800
7801             my $readycounter = 0;
7802
7803             for my $job_id (sort keys %$jobs) {
7804                 my $job = $running_jobs->{$job_id};
7805
7806                 my $vanished = !defined($job);
7807                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7808                 if($complete || ($vanished && $completion eq 'auto')) {
7809                     print "$job_id: $op-job finished\n";
7810                     delete $jobs->{$job_id};
7811                     next;
7812                 }
7813
7814                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7815
7816                 my $busy = $job->{busy};
7817                 my $ready = $job->{ready};
7818                 if (my $total = $job->{len}) {
7819                     my $transferred = $job->{offset} || 0;
7820                     my $remaining = $total - $transferred;
7821                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7822
7823                     my $duration = $ctime - $starttime;
7824                     my $total_h = render_bytes($total, 1);
7825                     my $transferred_h = render_bytes($transferred, 1);
7826
7827                     my $status = sprintf(
7828                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7829                         render_duration($duration),
7830                     );
7831
7832                     if ($ready) {
7833                         if ($busy) {
7834                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7835                         } else {
7836                             $status .= ", ready";
7837                         }
7838                     }
7839                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7840                     $jobs->{$job_id}->{ready} = $ready;
7841                 }
7842
7843                 $readycounter++ if $job->{ready};
7844             }
7845
7846             last if scalar(keys %$jobs) == 0;
7847
7848             if ($readycounter == scalar(keys %$jobs)) {
7849                 print "all '$op' jobs are ready\n";
7850
7851                 # do the complete later (or has already been done)
7852                 last if $completion eq 'skip' || $completion eq 'auto';
7853
7854                 if ($vmiddst && $vmiddst != $vmid) {
7855                     my $agent_running = $qga && qga_check_running($vmid);
7856                     if ($agent_running) {
7857                         print "freeze filesystem\n";
7858                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7859                         warn $@ if $@;
7860                     } else {
7861                         print "suspend vm\n";
7862                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7863                         warn $@ if $@;
7864                     }
7865
7866                     # if we clone a disk for a new target vm, we don't switch the disk
7867                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7868
7869                     if ($agent_running) {
7870                         print "unfreeze filesystem\n";
7871                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7872                         warn $@ if $@;
7873                     } else {
7874                         print "resume vm\n";
7875                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7876                         warn $@ if $@;
7877                     }
7878
7879                     last;
7880                 } else {
7881
7882                     for my $job_id (sort keys %$jobs) {
7883                         # try to switch the disk if source and destination are on the same guest
7884                         print "$job_id: Completing block job_id...\n";
7885
7886                         my $op;
7887                         if ($completion eq 'complete') {
7888                             $op = 'block-job-complete';
7889                         } elsif ($completion eq 'cancel') {
7890                             $op = 'block-job-cancel';
7891                         } else {
7892                             die "invalid completion value: $completion\n";
7893                         }
7894                         eval { mon_cmd($vmid, $op, device => $job_id) };
7895                         if ($@ =~ m/cannot be completed/) {
7896                             print "$job_id: block job cannot be completed, trying again.\n";
7897                             $err_complete++;
7898                         }else {
7899                             print "$job_id: Completed successfully.\n";
7900                             $jobs->{$job_id}->{complete} = 1;
7901                         }
7902                     }
7903                 }
7904             }
7905             sleep 1;
7906         }
7907     };
7908     my $err = $@;
7909
7910     if ($err) {
7911         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7912         die "block job ($op) error: $err";
7913     }
7914 }
7915
7916 sub qemu_blockjobs_cancel {
7917     my ($vmid, $jobs) = @_;
7918
7919     foreach my $job (keys %$jobs) {
7920         print "$job: Cancelling block job\n";
7921         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7922         $jobs->{$job}->{cancel} = 1;
7923     }
7924
7925     while (1) {
7926         my $stats = mon_cmd($vmid, "query-block-jobs");
7927
7928         my $running_jobs = {};
7929         foreach my $stat (@$stats) {
7930             $running_jobs->{$stat->{device}} = $stat;
7931         }
7932
7933         foreach my $job (keys %$jobs) {
7934
7935             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7936                 print "$job: Done.\n";
7937                 delete $jobs->{$job};
7938             }
7939         }
7940
7941         last if scalar(keys %$jobs) == 0;
7942
7943         sleep 1;
7944     }
7945 }
7946
7947 sub clone_disk {
7948     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7949
7950     my ($vmid, $running) = $source->@{qw(vmid running)};
7951     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7952
7953     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7954     my ($storage, $format) = $dest->@{qw(storage format)};
7955
7956     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7957
7958     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7959         die "cloning from/to EFI disk requires EFI disk\n"
7960             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7961         die "cloning from/to TPM state requires TPM state\n"
7962             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7963
7964         # This would lead to two device nodes in QEMU pointing to the same backing image!
7965         die "cannot change drive name when cloning disk from/to the same VM\n"
7966             if $use_drive_mirror && $vmid == $newvmid;
7967     }
7968
7969     die "cannot move TPM state while VM is running\n"
7970         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7971
7972     my $newvolid;
7973
7974     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7975     print "$src_drivename " if $src_drivename;
7976     print "($drive->{file})\n";
7977
7978     if (!$full) {
7979         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7980         push @$newvollist, $newvolid;
7981     } else {
7982
7983         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7984         $storeid = $storage if $storage;
7985
7986         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7987
7988         my $name = undef;
7989         my $size = undef;
7990         if (drive_is_cloudinit($drive)) {
7991             $name = "vm-$newvmid-cloudinit";
7992             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7993             if ($scfg->{path}) {
7994                 $name .= ".$dst_format";
7995             }
7996             $snapname = undef;
7997             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7998         } elsif ($dst_drivename eq 'efidisk0') {
7999             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8000         } elsif ($dst_drivename eq 'tpmstate0') {
8001             $dst_format = 'raw';
8002             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8003         } else {
8004             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8005         }
8006         $newvolid = PVE::Storage::vdisk_alloc(
8007             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8008         );
8009         push @$newvollist, $newvolid;
8010
8011         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8012
8013         if (drive_is_cloudinit($drive)) {
8014             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8015             # if this is the case, we have to complete any block-jobs still there from
8016             # previous drive-mirrors
8017             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8018                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8019             }
8020             goto no_data_clone;
8021         }
8022
8023         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8024         if ($use_drive_mirror) {
8025             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8026                 $completion, $qga, $bwlimit);
8027         } else {
8028             # TODO: handle bwlimits
8029             if ($dst_drivename eq 'efidisk0') {
8030                 # the relevant data on the efidisk may be smaller than the source
8031                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8032                 # that is given by the OVMF_VARS.fd
8033                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8034                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8035
8036                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8037
8038                 # better for Ceph if block size is not too small, see bug #3324
8039                 my $bs = 1024*1024;
8040
8041                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8042
8043                 if ($src_format eq 'qcow2' && $snapname) {
8044                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8045                         if !min_version(kvm_user_version(), 6, 2);
8046                     push $cmd->@*, '-l', $snapname;
8047                 }
8048                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8049                 run_command($cmd);
8050             } else {
8051                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
8052             }
8053         }
8054     }
8055
8056 no_data_clone:
8057     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8058
8059     my $disk = dclone($drive);
8060     delete $disk->{format};
8061     $disk->{file} = $newvolid;
8062     $disk->{size} = $size if defined($size);
8063
8064     return $disk;
8065 }
8066
8067 sub get_running_qemu_version {
8068     my ($vmid) = @_;
8069     my $res = mon_cmd($vmid, "query-version");
8070     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8071 }
8072
8073 sub qemu_use_old_bios_files {
8074     my ($machine_type) = @_;
8075
8076     return if !$machine_type;
8077
8078     my $use_old_bios_files = undef;
8079
8080     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8081         $machine_type = $1;
8082         $use_old_bios_files = 1;
8083     } else {
8084         my $version = extract_version($machine_type, kvm_user_version());
8085         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8086         # load new efi bios files on migration. So this hack is required to allow
8087         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8088         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8089         $use_old_bios_files = !min_version($version, 2, 4);
8090     }
8091
8092     return ($use_old_bios_files, $machine_type);
8093 }
8094
8095 sub get_efivars_size {
8096     my ($conf, $efidisk) = @_;
8097
8098     my $arch = get_vm_arch($conf);
8099     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8100     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8101     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8102     return -s $ovmf_vars;
8103 }
8104
8105 sub update_efidisk_size {
8106     my ($conf) = @_;
8107
8108     return if !defined($conf->{efidisk0});
8109
8110     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8111     $disk->{size} = get_efivars_size($conf);
8112     $conf->{efidisk0} = print_drive($disk);
8113
8114     return;
8115 }
8116
8117 sub update_tpmstate_size {
8118     my ($conf) = @_;
8119
8120     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8121     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8122     $conf->{tpmstate0} = print_drive($disk);
8123 }
8124
8125 sub create_efidisk($$$$$$$) {
8126     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8127
8128     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8129
8130     my $vars_size_b = -s $ovmf_vars;
8131     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8132     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8133     PVE::Storage::activate_volumes($storecfg, [$volid]);
8134
8135     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8136     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8137
8138     return ($volid, $size/1024);
8139 }
8140
8141 sub vm_iothreads_list {
8142     my ($vmid) = @_;
8143
8144     my $res = mon_cmd($vmid, 'query-iothreads');
8145
8146     my $iothreads = {};
8147     foreach my $iothread (@$res) {
8148         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8149     }
8150
8151     return $iothreads;
8152 }
8153
8154 sub scsihw_infos {
8155     my ($conf, $drive) = @_;
8156
8157     my $maxdev = 0;
8158
8159     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8160         $maxdev = 7;
8161     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8162         $maxdev = 1;
8163     } else {
8164         $maxdev = 256;
8165     }
8166
8167     my $controller = int($drive->{index} / $maxdev);
8168     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8169         ? "virtioscsi"
8170         : "scsihw";
8171
8172     return ($maxdev, $controller, $controller_prefix);
8173 }
8174
8175 sub resolve_dst_disk_format {
8176         my ($storecfg, $storeid, $src_volname, $format) = @_;
8177         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8178
8179         if (!$format) {
8180             # if no target format is specified, use the source disk format as hint
8181             if ($src_volname) {
8182                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8183                 $format = qemu_img_format($scfg, $src_volname);
8184             } else {
8185                 return $defFormat;
8186             }
8187         }
8188
8189         # test if requested format is supported - else use default
8190         my $supported = grep { $_ eq $format } @$validFormats;
8191         $format = $defFormat if !$supported;
8192         return $format;
8193 }
8194
8195 # NOTE: if this logic changes, please update docs & possibly gui logic
8196 sub find_vmstate_storage {
8197     my ($conf, $storecfg) = @_;
8198
8199     # first, return storage from conf if set
8200     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8201
8202     my ($target, $shared, $local);
8203
8204     foreach_storage_used_by_vm($conf, sub {
8205         my ($sid) = @_;
8206         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8207         my $dst = $scfg->{shared} ? \$shared : \$local;
8208         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8209     });
8210
8211     # second, use shared storage where VM has at least one disk
8212     # third, use local storage where VM has at least one disk
8213     # fall back to local storage
8214     $target = $shared // $local // 'local';
8215
8216     return $target;
8217 }
8218
8219 sub generate_uuid {
8220     my ($uuid, $uuid_str);
8221     UUID::generate($uuid);
8222     UUID::unparse($uuid, $uuid_str);
8223     return $uuid_str;
8224 }
8225
8226 sub generate_smbios1_uuid {
8227     return "uuid=".generate_uuid();
8228 }
8229
8230 sub nbd_stop {
8231     my ($vmid) = @_;
8232
8233     mon_cmd($vmid, 'nbd-server-stop');
8234 }
8235
8236 sub create_reboot_request {
8237     my ($vmid) = @_;
8238     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8239         or die "failed to create reboot trigger file: $!\n";
8240     close($fh);
8241 }
8242
8243 sub clear_reboot_request {
8244     my ($vmid) = @_;
8245     my $path = "/run/qemu-server/$vmid.reboot";
8246     my $res = 0;
8247
8248     $res = unlink($path);
8249     die "could not remove reboot request for $vmid: $!"
8250         if !$res && $! != POSIX::ENOENT;
8251
8252     return $res;
8253 }
8254
8255 sub bootorder_from_legacy {
8256     my ($conf, $bootcfg) = @_;
8257
8258     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8259     my $bootindex_hash = {};
8260     my $i = 1;
8261     foreach my $o (split(//, $boot)) {
8262         $bootindex_hash->{$o} = $i*100;
8263         $i++;
8264     }
8265
8266     my $bootorder = {};
8267
8268     PVE::QemuConfig->foreach_volume($conf, sub {
8269         my ($ds, $drive) = @_;
8270
8271         if (drive_is_cdrom ($drive, 1)) {
8272             if ($bootindex_hash->{d}) {
8273                 $bootorder->{$ds} = $bootindex_hash->{d};
8274                 $bootindex_hash->{d} += 1;
8275             }
8276         } elsif ($bootindex_hash->{c}) {
8277             $bootorder->{$ds} = $bootindex_hash->{c}
8278                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8279             $bootindex_hash->{c} += 1;
8280         }
8281     });
8282
8283     if ($bootindex_hash->{n}) {
8284         for (my $i = 0; $i < $MAX_NETS; $i++) {
8285             my $netname = "net$i";
8286             next if !$conf->{$netname};
8287             $bootorder->{$netname} = $bootindex_hash->{n};
8288             $bootindex_hash->{n} += 1;
8289         }
8290     }
8291
8292     return $bootorder;
8293 }
8294
8295 # Generate default device list for 'boot: order=' property. Matches legacy
8296 # default boot order, but with explicit device names. This is important, since
8297 # the fallback for when neither 'order' nor the old format is specified relies
8298 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8299 sub get_default_bootdevices {
8300     my ($conf) = @_;
8301
8302     my @ret = ();
8303
8304     # harddisk
8305     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8306     push @ret, $first if $first;
8307
8308     # cdrom
8309     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8310     push @ret, $first if $first;
8311
8312     # network
8313     for (my $i = 0; $i < $MAX_NETS; $i++) {
8314         my $netname = "net$i";
8315         next if !$conf->{$netname};
8316         push @ret, $netname;
8317         last;
8318     }
8319
8320     return \@ret;
8321 }
8322
8323 sub device_bootorder {
8324     my ($conf) = @_;
8325
8326     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8327
8328     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8329
8330     my $bootorder = {};
8331     if (!defined($boot) || $boot->{legacy}) {
8332         $bootorder = bootorder_from_legacy($conf, $boot);
8333     } elsif ($boot->{order}) {
8334         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8335         for my $dev (PVE::Tools::split_list($boot->{order})) {
8336             $bootorder->{$dev} = $i++;
8337         }
8338     }
8339
8340     return $bootorder;
8341 }
8342
8343 sub register_qmeventd_handle {
8344     my ($vmid) = @_;
8345
8346     my $fh;
8347     my $peer = "/var/run/qmeventd.sock";
8348     my $count = 0;
8349
8350     for (;;) {
8351         $count++;
8352         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8353         last if $fh;
8354         if ($! != EINTR && $! != EAGAIN) {
8355             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8356         }
8357         if ($count > 4) {
8358             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8359               . "after $count retries\n";
8360         }
8361         usleep(25000);
8362     }
8363
8364     # send handshake to mark VM as backing up
8365     print $fh to_json({vzdump => {vmid => "$vmid"}});
8366
8367     # return handle to be closed later when inhibit is no longer required
8368     return $fh;
8369 }
8370
8371 # bash completion helper
8372
8373 sub complete_backup_archives {
8374     my ($cmdname, $pname, $cvalue) = @_;
8375
8376     my $cfg = PVE::Storage::config();
8377
8378     my $storeid;
8379
8380     if ($cvalue =~ m/^([^:]+):/) {
8381         $storeid = $1;
8382     }
8383
8384     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8385
8386     my $res = [];
8387     foreach my $id (keys %$data) {
8388         foreach my $item (@{$data->{$id}}) {
8389             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8390             push @$res, $item->{volid} if defined($item->{volid});
8391         }
8392     }
8393
8394     return $res;
8395 }
8396
8397 my $complete_vmid_full = sub {
8398     my ($running) = @_;
8399
8400     my $idlist = vmstatus();
8401
8402     my $res = [];
8403
8404     foreach my $id (keys %$idlist) {
8405         my $d = $idlist->{$id};
8406         if (defined($running)) {
8407             next if $d->{template};
8408             next if $running && $d->{status} ne 'running';
8409             next if !$running && $d->{status} eq 'running';
8410         }
8411         push @$res, $id;
8412
8413     }
8414     return $res;
8415 };
8416
8417 sub complete_vmid {
8418     return &$complete_vmid_full();
8419 }
8420
8421 sub complete_vmid_stopped {
8422     return &$complete_vmid_full(0);
8423 }
8424
8425 sub complete_vmid_running {
8426     return &$complete_vmid_full(1);
8427 }
8428
8429 sub complete_storage {
8430
8431     my $cfg = PVE::Storage::config();
8432     my $ids = $cfg->{ids};
8433
8434     my $res = [];
8435     foreach my $sid (keys %$ids) {
8436         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8437         next if !$ids->{$sid}->{content}->{images};
8438         push @$res, $sid;
8439     }
8440
8441     return $res;
8442 }
8443
8444 sub complete_migration_storage {
8445     my ($cmd, $param, $current_value, $all_args) = @_;
8446
8447     my $targetnode = @$all_args[1];
8448
8449     my $cfg = PVE::Storage::config();
8450     my $ids = $cfg->{ids};
8451
8452     my $res = [];
8453     foreach my $sid (keys %$ids) {
8454         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8455         next if !$ids->{$sid}->{content}->{images};
8456         push @$res, $sid;
8457     }
8458
8459     return $res;
8460 }
8461
8462 sub vm_is_paused {
8463     my ($vmid) = @_;
8464     my $qmpstatus = eval {
8465         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8466         mon_cmd($vmid, "query-status");
8467     };
8468     warn "$@\n" if $@;
8469     return $qmpstatus && $qmpstatus->{status} eq "paused";
8470 }
8471
8472 sub check_volume_storage_type {
8473     my ($storecfg, $vol) = @_;
8474
8475     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8476     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8477     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8478
8479     die "storage '$storeid' does not support content-type '$vtype'\n"
8480         if !$scfg->{content}->{$vtype};
8481
8482     return 1;
8483 }
8484
8485 sub add_nets_bridge_fdb {
8486     my ($conf, $vmid) = @_;
8487
8488     for my $opt (keys %$conf) {
8489         next if $opt !~ m/^net(\d+)$/;
8490         my $iface = "tap${vmid}i$1";
8491         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8492         my $net = parse_net($conf->{$opt}, 1) or next;
8493
8494         my $mac = $net->{macaddr};
8495         if (!$mac) {
8496             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8497                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8498             next;
8499         }
8500
8501         my $bridge = $net->{bridge};
8502         if ($have_sdn) {
8503             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8504         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8505             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8506         }
8507     }
8508 }
8509
8510 sub del_nets_bridge_fdb {
8511     my ($conf, $vmid) = @_;
8512
8513     for my $opt (keys %$conf) {
8514         next if $opt !~ m/^net(\d+)$/;
8515         my $iface = "tap${vmid}i$1";
8516
8517         my $net = parse_net($conf->{$opt}) or next;
8518         my $mac = $net->{macaddr} or next;
8519
8520         my $bridge = $net->{bridge};
8521         if ($have_sdn) {
8522             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8523         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8524             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8525         }
8526     }
8527 }
8528
8529 1;