PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::INotify;
  38 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  39 use PVE::ProcFSTools;
  40 use PVE::PBSClient;
  41 use PVE::RESTEnvironment qw(log_warn);
  42 use PVE::RPCEnvironment;
  43 use PVE::Storage;
  44 use PVE::SysFSTools;
  45 use PVE::Systemd;
  46 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  47
  48 use PVE::QMPClient;
  49 use PVE::QemuConfig;
  50 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  51 use PVE::QemuServer::Cloudinit;
  52 use PVE::QemuServer::CGroup;
  53 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  54 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  55 use PVE::QemuServer::Machine;
  56 use PVE::QemuServer::Memory;
  57 use PVE::QemuServer::Monitor qw(mon_cmd);
  58 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  59 use PVE::QemuServer::USB qw(parse_usb_device);
  60
  61 my $have_sdn;
  62 eval {
  63     require PVE::Network::SDN::Zones;
  64     $have_sdn = 1;
  65 };
  66
  67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  68 my $OVMF = {
  69     x86_64 => {
  70         '4m-no-smm' => [
  71             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  72             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  73         ],
  74         '4m-no-smm-ms' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  77         ],
  78         '4m' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  81         ],
  82         '4m-ms' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  85         ],
  86         default => [
  87             "$EDK2_FW_BASE/OVMF_CODE.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS.fd",
  89         ],
  90     },
  91     aarch64 => {
  92         default => [
  93             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  94             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  95         ],
  96     },
  97 };
  98
  99 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 100
 101 # Note about locking: we use flock on the config file protect against concurent actions.
 102 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 104 # But you can ignore this kind of lock with the --skiplock flag.
 105
 106 cfs_register_file(
 107     '/qemu-server/',
 108     \&parse_vm_config,
 109     \&write_vm_config
 110 );
 111
 112 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 113     description => "Some command save/restore state from this location.",
 114     type => 'string',
 115     maxLength => 128,
 116     optional => 1,
 117 });
 118
 119 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 120         description => "Specifies the QEMU machine type.",
 121         type => 'string',
 122         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 123         maxLength => 40,
 124         optional => 1,
 125 });
 126
 127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 244             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 245             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 246             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 247             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 248             ." a hardware RNG from the host.",
 249     },
 250     max_bytes => {
 251         type => 'integer',
 252         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 253             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 254             ." `0` to disable limiting (potentially dangerous!).",
 255         optional => 1,
 256
 257         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 258         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 259         # reading from /dev/urandom
 260         default => 1024,
 261     },
 262     period => {
 263         type => 'integer',
 264         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 265             ." the guest to retrieve another 'max_bytes' of entropy.",
 266         optional => 1,
 267         default => 1000,
 268     },
 269 };
 270
 271 my $meta_info_fmt = {
 272     'ctime' => {
 273         type => 'integer',
 274         description => "The guest creation timestamp as UNIX epoch time",
 275         minimum => 0,
 276         optional => 1,
 277     },
 278     'creation-qemu' => {
 279         type => 'string',
 280         description => "The QEMU (machine) version from the time this VM was created.",
 281         pattern => '\d+(\.\d+)+',
 282         optional => 1,
 283     },
 284 };
 285
 286 my $confdesc = {
 287     onboot => {
 288         optional => 1,
 289         type => 'boolean',
 290         description => "Specifies whether a VM will be started during system bootup.",
 291         default => 0,
 292     },
 293     autostart => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Automatic restart after crash (currently ignored).",
 297         default => 0,
 298     },
 299     hotplug => {
 300         optional => 1,
 301         type => 'string', format => 'pve-hotplug-features',
 302         description => "Selectively enable hotplug features. This is a comma separated list of"
 303             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 304             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 305             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 306             ." windows > 7.",
 307         default => 'network,disk,usb',
 308     },
 309     reboot => {
 310         optional => 1,
 311         type => 'boolean',
 312         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 313         default => 1,
 314     },
 315     lock => {
 316         optional => 1,
 317         type => 'string',
 318         description => "Lock/unlock the VM.",
 319         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 320     },
 321     cpulimit => {
 322         optional => 1,
 323         type => 'number',
 324         description => "Limit of CPU usage.",
 325         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 326             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 327         minimum => 0,
 328         maximum => 128,
 329         default => 0,
 330     },
 331     cpuunits => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 335         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 336             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 337             ." weights of all the other running VMs.",
 338         minimum => 1,
 339         maximum => 262144,
 340         default => 'cgroup v1: 1024, cgroup v2: 100',
 341     },
 342     memory => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 346             ." you use the balloon device.",
 347         minimum => 16,
 348         default => 512,
 349     },
 350     balloon => {
 351         optional => 1,
 352         type => 'integer',
 353         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 354         minimum => 0,
 355     },
 356     shares => {
 357         optional => 1,
 358         type => 'integer',
 359         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 360             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 361             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 362         minimum => 0,
 363         maximum => 50000,
 364         default => 1000,
 365     },
 366     keyboard => {
 367         optional => 1,
 368         type => 'string',
 369         description => "Keyboard layout for VNC server. This option is generally not required and"
 370         ." is often better handled from within the guest OS.",
 371         enum => PVE::Tools::kvmkeymaplist(),
 372         default => undef,
 373     },
 374     name => {
 375         optional => 1,
 376         type => 'string', format => 'dns-name',
 377         description => "Set a name for the VM. Only used on the configuration web interface.",
 378     },
 379     scsihw => {
 380         optional => 1,
 381         type => 'string',
 382         description => "SCSI controller model",
 383         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 384         default => 'lsi',
 385     },
 386     description => {
 387         optional => 1,
 388         type => 'string',
 389         description => "Description for the VM. Shown in the web-interface VM's summary."
 390             ." This is saved as comment inside the configuration file.",
 391         maxLength => 1024 * 8,
 392     },
 393     ostype => {
 394         optional => 1,
 395         type => 'string',
 396         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 397         description => "Specify guest operating system.",
 398         verbose_description => <<EODESC,
 399 Specify guest operating system. This is used to enable special
 400 optimization/features for specific operating systems:
 401
 402 [horizontal]
 403 other;; unspecified OS
 404 wxp;; Microsoft Windows XP
 405 w2k;; Microsoft Windows 2000
 406 w2k3;; Microsoft Windows 2003
 407 w2k8;; Microsoft Windows 2008
 408 wvista;; Microsoft Windows Vista
 409 win7;; Microsoft Windows 7
 410 win8;; Microsoft Windows 8/2012/2012r2
 411 win10;; Microsoft Windows 10/2016/2019
 412 win11;; Microsoft Windows 11/2022
 413 l24;; Linux 2.4 Kernel
 414 l26;; Linux 2.6 - 6.X Kernel
 415 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 416 EODESC
 417     },
 418     boot => {
 419         optional => 1,
 420         type => 'string', format => 'pve-qm-boot',
 421         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 422             ." key or 'legacy=' is deprecated.",
 423     },
 424     bootdisk => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-bootdisk',
 427         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 428         pattern => '(ide|sata|scsi|virtio)\d+',
 429     },
 430     smp => {
 431         optional => 1,
 432         type => 'integer',
 433         description => "The number of CPUs. Please use option -sockets instead.",
 434         minimum => 1,
 435         default => 1,
 436     },
 437     sockets => {
 438         optional => 1,
 439         type => 'integer',
 440         description => "The number of CPU sockets.",
 441         minimum => 1,
 442         default => 1,
 443     },
 444     cores => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "The number of cores per socket.",
 448         minimum => 1,
 449         default => 1,
 450     },
 451     numa => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable NUMA.",
 455         default => 0,
 456     },
 457     hugepages => {
 458         optional => 1,
 459         type => 'string',
 460         description => "Enable/disable hugepages memory.",
 461         enum => [qw(any 2 1024)],
 462     },
 463     keephugepages => {
 464         optional => 1,
 465         type => 'boolean',
 466         default => 0,
 467         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 468             ." after VM shutdown and can be used for subsequent starts.",
 469     },
 470     vcpus => {
 471         optional => 1,
 472         type => 'integer',
 473         description => "Number of hotplugged vcpus.",
 474         minimum => 1,
 475         default => 0,
 476     },
 477     acpi => {
 478         optional => 1,
 479         type => 'boolean',
 480         description => "Enable/disable ACPI.",
 481         default => 1,
 482     },
 483     agent => {
 484         optional => 1,
 485         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 486         type => 'string',
 487         format => $agent_fmt,
 488     },
 489     kvm => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Enable/disable KVM hardware virtualization.",
 493         default => 1,
 494     },
 495     tdf => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable time drift fix.",
 499         default => 0,
 500     },
 501     localtime => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 505             ." the `ostype` indicates a Microsoft Windows OS.",
 506     },
 507     freeze => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 511     },
 512     vga => {
 513         optional => 1,
 514         type => 'string', format => $vga_fmt,
 515         description => "Configure the VGA hardware.",
 516         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 517             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 518             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 519             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 520             ." display server. For win* OS you can select how many independent displays you want,"
 521             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 522             ." using a serial device as terminal.",
 523     },
 524     watchdog => {
 525         optional => 1,
 526         type => 'string', format => 'pve-qm-watchdog',
 527         description => "Create a virtual hardware watchdog device.",
 528         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 529             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 530             ." else the watchdog will reset the guest (or execute the respective action specified)",
 531     },
 532     startdate => {
 533         optional => 1,
 534         type => 'string',
 535         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 536         description => "Set the initial date of the real time clock. Valid format for date are:"
 537             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 538         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 539         default => 'now',
 540     },
 541     startup =>  get_standard_option('pve-startup-order'),
 542     template => {
 543         optional => 1,
 544         type => 'boolean',
 545         description => "Enable/disable Template.",
 546         default => 0,
 547     },
 548     args => {
 549         optional => 1,
 550         type => 'string',
 551         description => "Arbitrary arguments passed to kvm.",
 552         verbose_description => <<EODESCR,
 553 Arbitrary arguments passed to kvm, for example:
 554
 555 args: -no-reboot -no-hpet
 556
 557 NOTE: this option is for experts only.
 558 EODESCR
 559     },
 560     tablet => {
 561         optional => 1,
 562         type => 'boolean',
 563         default => 1,
 564         description => "Enable/disable the USB tablet device.",
 565         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 566             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 567             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 568             ." may consider disabling this to save some context switches. This is turned off by"
 569             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 570     },
 571     migrate_speed => {
 572         optional => 1,
 573         type => 'integer',
 574         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 575         minimum => 0,
 576         default => 0,
 577     },
 578     migrate_downtime => {
 579         optional => 1,
 580         type => 'number',
 581         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 582         minimum => 0,
 583         default => 0.1,
 584     },
 585     cdrom => {
 586         optional => 1,
 587         type => 'string', format => 'pve-qm-ide',
 588         typetext => '<volume>',
 589         description => "This is an alias for option -ide2",
 590     },
 591     cpu => {
 592         optional => 1,
 593         description => "Emulated CPU type.",
 594         type => 'string',
 595         format => 'pve-vm-cpu-conf',
 596     },
 597     parent => get_standard_option('pve-snapshot-name', {
 598         optional => 1,
 599         description => "Parent snapshot name. This is used internally, and should not be modified.",
 600     }),
 601     snaptime => {
 602         optional => 1,
 603         description => "Timestamp for snapshots.",
 604         type => 'integer',
 605         minimum => 0,
 606     },
 607     vmstate => {
 608         optional => 1,
 609         type => 'string', format => 'pve-volume-id',
 610         description => "Reference to a volume which stores the VM state. This is used internally"
 611             ." for snapshots.",
 612     },
 613     vmstatestorage => get_standard_option('pve-storage-id', {
 614         description => "Default storage for VM state volumes/files.",
 615         optional => 1,
 616     }),
 617     runningmachine => get_standard_option('pve-qemu-machine', {
 618         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 619             ." for snapshots.",
 620     }),
 621     runningcpu => {
 622         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 623             ." internally for snapshots.",
 624         optional => 1,
 625         type => 'string',
 626         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 627         format_description => 'QEMU -cpu parameter'
 628     },
 629     machine => get_standard_option('pve-qemu-machine'),
 630     arch => {
 631         description => "Virtual processor architecture. Defaults to the host.",
 632         optional => 1,
 633         type => 'string',
 634         enum => [qw(x86_64 aarch64)],
 635     },
 636     smbios1 => {
 637         description => "Specify SMBIOS type 1 fields.",
 638         type => 'string', format => 'pve-qm-smbios1',
 639         maxLength => 512,
 640         optional => 1,
 641     },
 642     protection => {
 643         optional => 1,
 644         type => 'boolean',
 645         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 646             ." remove disk operations.",
 647         default => 0,
 648     },
 649     bios => {
 650         optional => 1,
 651         type => 'string',
 652         enum => [ qw(seabios ovmf) ],
 653         description => "Select BIOS implementation.",
 654         default => 'seabios',
 655     },
 656     vmgenid => {
 657         type => 'string',
 658         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 659         format_description => 'UUID',
 660         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 661             ." to disable explicitly.",
 662         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 663             ." value identifier to the guest OS. This allows to notify the guest operating system"
 664             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 665             ." execution or creation from a template). The guest operating system notices the"
 666             ." change, and is then able to react as appropriate by marking its copies of"
 667             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 668             ."Note that auto-creation only works when done through API/CLI create or update methods"
 669             .", but not when manually editing the config file.",
 670         default => "1 (autogenerated)",
 671         optional => 1,
 672     },
 673     hookscript => {
 674         type => 'string',
 675         format => 'pve-volume-id',
 676         optional => 1,
 677         description => "Script that will be executed during various steps in the vms lifetime.",
 678     },
 679     ivshmem => {
 680         type => 'string',
 681         format => $ivshmem_fmt,
 682         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 683             ." the host.",
 684         optional => 1,
 685     },
 686     audio0 => {
 687         type => 'string',
 688         format => $audio_fmt,
 689         description => "Configure a audio device, useful in combination with QXL/Spice.",
 690         optional => 1
 691     },
 692     spice_enhancements => {
 693         type => 'string',
 694         format => $spice_enhancements_fmt,
 695         description => "Configure additional enhancements for SPICE.",
 696         optional => 1
 697     },
 698     tags => {
 699         type => 'string', format => 'pve-tag-list',
 700         description => 'Tags of the VM. This is only meta information.',
 701         optional => 1,
 702     },
 703     rng0 => {
 704         type => 'string',
 705         format => $rng_fmt,
 706         description => "Configure a VirtIO-based Random Number Generator.",
 707         optional => 1,
 708     },
 709     meta => {
 710         type => 'string',
 711         format => $meta_info_fmt,
 712         description => "Some (read-only) meta-information about this guest.",
 713         optional => 1,
 714     },
 715     affinity => {
 716         type => 'string', format => 'pve-cpuset',
 717         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 718         optional => 1,
 719     },
 720 };
 721
 722 my $cicustom_fmt = {
 723     meta => {
 724         type => 'string',
 725         optional => 1,
 726         description => 'Specify a custom file containing all meta data passed to the VM via"
 727             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731     network => {
 732         type => 'string',
 733         optional => 1,
 734         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 735         format => 'pve-volume-id',
 736         format_description => 'volume',
 737     },
 738     user => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 742         format => 'pve-volume-id',
 743         format_description => 'volume',
 744     },
 745     vendor => {
 746         type => 'string',
 747         optional => 1,
 748         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 749         format => 'pve-volume-id',
 750         format_description => 'volume',
 751     },
 752 };
 753 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 754
 755 my $confdesc_cloudinit = {
 756     citype => {
 757         optional => 1,
 758         type => 'string',
 759         description => 'Specifies the cloud-init configuration format. The default depends on the'
 760             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 761             .' and `configdrive2` for windows.',
 762         enum => ['configdrive2', 'nocloud', 'opennebula'],
 763     },
 764     ciuser => {
 765         optional => 1,
 766         type => 'string',
 767         description => "cloud-init: User name to change ssh keys and password for instead of the"
 768             ." image's configured default user.",
 769     },
 770     cipassword => {
 771         optional => 1,
 772         type => 'string',
 773         description => 'cloud-init: Password to assign the user. Using this is generally not'
 774             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 775             .' support hashed passwords.',
 776     },
 777     cicustom => {
 778         optional => 1,
 779         type => 'string',
 780         description => 'cloud-init: Specify custom files to replace the automatically generated'
 781             .' ones at start.',
 782         format => 'pve-qm-cicustom',
 783     },
 784     searchdomain => {
 785         optional => 1,
 786         type => 'string',
 787         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 788             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 789             .' are set.',
 790     },
 791     nameserver => {
 792         optional => 1,
 793         type => 'string', format => 'address-list',
 794         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 795             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 796             .' are set.',
 797     },
 798     sshkeys => {
 799         optional => 1,
 800         type => 'string',
 801         format => 'urlencoded',
 802         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 803     },
 804 };
 805
 806 # what about other qemu settings ?
 807 #cpu => 'string',
 808 #machine => 'string',
 809 #fda => 'file',
 810 #fdb => 'file',
 811 #mtdblock => 'file',
 812 #sd => 'file',
 813 #pflash => 'file',
 814 #snapshot => 'bool',
 815 #bootp => 'file',
 816 ##tftp => 'dir',
 817 ##smb => 'dir',
 818 #kernel => 'file',
 819 #append => 'string',
 820 #initrd => 'file',
 821 ##soundhw => 'string',
 822
 823 while (my ($k, $v) = each %$confdesc) {
 824     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 825 }
 826
 827 my $MAX_USB_DEVICES = 14;
 828 my $MAX_NETS = 32;
 829 my $MAX_SERIAL_PORTS = 4;
 830 my $MAX_PARALLEL_PORTS = 3;
 831 my $MAX_NUMA = 8;
 832
 833 my $numa_fmt = {
 834     cpus => {
 835         type => "string",
 836         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 837         description => "CPUs accessing this NUMA node.",
 838         format_description => "id[-id];...",
 839     },
 840     memory => {
 841         type => "number",
 842         description => "Amount of memory this NUMA node provides.",
 843         optional => 1,
 844     },
 845     hostnodes => {
 846         type => "string",
 847         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 848         description => "Host NUMA nodes to use.",
 849         format_description => "id[-id];...",
 850         optional => 1,
 851     },
 852     policy => {
 853         type => 'string',
 854         enum => [qw(preferred bind interleave)],
 855         description => "NUMA allocation policy.",
 856         optional => 1,
 857     },
 858 };
 859 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 860 my $numadesc = {
 861     optional => 1,
 862     type => 'string', format => $numa_fmt,
 863     description => "NUMA topology.",
 864 };
 865 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 866
 867 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 868     $confdesc->{"numa$i"} = $numadesc;
 869 }
 870
 871 my $nic_model_list = [
 872     'e1000',
 873     'e1000-82540em',
 874     'e1000-82544gc',
 875     'e1000-82545em',
 876     'e1000e',
 877     'i82551',
 878     'i82557b',
 879     'i82559er',
 880     'ne2k_isa',
 881     'ne2k_pci',
 882     'pcnet',
 883     'rtl8139',
 884     'virtio',
 885     'vmxnet3',
 886 ];
 887 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 888
 889 my $net_fmt_bridge_descr = <<__EOD__;
 890 Bridge to attach the network device to. The Proxmox VE standard bridge
 891 is called 'vmbr0'.
 892
 893 If you do not specify a bridge, we create a kvm user (NATed) network
 894 device, which provides DHCP and DNS services. The following addresses
 895 are used:
 896
 897  10.0.2.2   Gateway
 898  10.0.2.3   DNS Server
 899  10.0.2.4   SMB Server
 900
 901 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 902 __EOD__
 903
 904 my $net_fmt = {
 905     macaddr  => get_standard_option('mac-addr', {
 906         description => "MAC address. That address must be unique withing your network. This is"
 907             ." automatically generated if not specified.",
 908     }),
 909     model => {
 910         type => 'string',
 911         description => "Network Card Model. The 'virtio' model provides the best performance with"
 912             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 913             ." best to use 'e1000'.",
 914         enum => $nic_model_list,
 915         default_key => 1,
 916     },
 917     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 918     bridge => get_standard_option('pve-bridge-id', {
 919         description => $net_fmt_bridge_descr,
 920         optional => 1,
 921     }),
 922     queues => {
 923         type => 'integer',
 924         minimum => 0, maximum => 64,
 925         description => 'Number of packet queues to be used on the device.',
 926         optional => 1,
 927     },
 928     rate => {
 929         type => 'number',
 930         minimum => 0,
 931         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 932         optional => 1,
 933     },
 934     tag => {
 935         type => 'integer',
 936         minimum => 1, maximum => 4094,
 937         description => 'VLAN tag to apply to packets on this interface.',
 938         optional => 1,
 939     },
 940     trunks => {
 941         type => 'string',
 942         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 943         description => 'VLAN trunks to pass through this interface.',
 944         format_description => 'vlanid[;vlanid...]',
 945         optional => 1,
 946     },
 947     firewall => {
 948         type => 'boolean',
 949         description => 'Whether this interface should be protected by the firewall.',
 950         optional => 1,
 951     },
 952     link_down => {
 953         type => 'boolean',
 954         description => 'Whether this interface should be disconnected (like pulling the plug).',
 955         optional => 1,
 956     },
 957     mtu => {
 958         type => 'integer',
 959         minimum => 1, maximum => 65520,
 960         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 961         optional => 1,
 962     },
 963 };
 964
 965 my $netdesc = {
 966     optional => 1,
 967     type => 'string', format => $net_fmt,
 968     description => "Specify network devices.",
 969 };
 970
 971 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 972
 973 my $ipconfig_fmt = {
 974     ip => {
 975         type => 'string',
 976         format => 'pve-ipv4-config',
 977         format_description => 'IPv4Format/CIDR',
 978         description => 'IPv4 address in CIDR format.',
 979         optional => 1,
 980         default => 'dhcp',
 981     },
 982     gw => {
 983         type => 'string',
 984         format => 'ipv4',
 985         format_description => 'GatewayIPv4',
 986         description => 'Default gateway for IPv4 traffic.',
 987         optional => 1,
 988         requires => 'ip',
 989     },
 990     ip6 => {
 991         type => 'string',
 992         format => 'pve-ipv6-config',
 993         format_description => 'IPv6Format/CIDR',
 994         description => 'IPv6 address in CIDR format.',
 995         optional => 1,
 996         default => 'dhcp',
 997     },
 998     gw6 => {
 999         type => 'string',
1000         format => 'ipv6',
1001         format_description => 'GatewayIPv6',
1002         description => 'Default gateway for IPv6 traffic.',
1003         optional => 1,
1004         requires => 'ip6',
1005     },
1006 };
1007 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1008 my $ipconfigdesc = {
1009     optional => 1,
1010     type => 'string', format => 'pve-qm-ipconfig',
1011     description => <<'EODESCR',
1012 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1013
1014 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1015
1016 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1017 gateway should be provided.
1018 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1019 cloud-init 19.4 or newer.
1020
1021 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1022 dhcp on IPv4.
1023 EODESCR
1024 };
1025 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1026
1027 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1028     $confdesc->{"net$i"} = $netdesc;
1029     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1030 }
1031
1032 foreach my $key (keys %$confdesc_cloudinit) {
1033     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1034 }
1035
1036 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1037 sub pve_verify_cpuset {
1038     my ($set_text, $noerr) = @_;
1039
1040     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1041
1042     if ($@) {
1043         return if $noerr;
1044         die "unable to parse cpuset option\n";
1045     }
1046
1047     return PVE::CpuSet->new($members)->short_string();
1048 }
1049
1050 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1051 sub verify_volume_id_or_qm_path {
1052     my ($volid, $noerr) = @_;
1053
1054     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1055
1056     return verify_volume_id_or_absolute_path($volid, $noerr);
1057 }
1058
1059 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1060 sub verify_volume_id_or_absolute_path {
1061     my ($volid, $noerr) = @_;
1062
1063     return $volid if $volid =~ m|^/|;
1064
1065     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1066     if ($@) {
1067         return if $noerr;
1068         die $@;
1069     }
1070     return $volid;
1071 }
1072
1073 my $usb_fmt = {
1074     host => {
1075         default_key => 1,
1076         type => 'string', format => 'pve-qm-usb-device',
1077         format_description => 'HOSTUSBDEVICE|spice',
1078         description => <<EODESCR,
1079 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1080
1081  'bus-port(.port)*' (decimal numbers) or
1082  'vendor_id:product_id' (hexadeciaml numbers) or
1083  'spice'
1084
1085 You can use the 'lsusb -t' command to list existing usb devices.
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1089
1090 The value 'spice' can be used to add a usb redirection devices for spice.
1091 EODESCR
1092     },
1093     usb3 => {
1094         optional => 1,
1095         type => 'boolean',
1096         description => "Specifies whether if given host option is a USB3 device or port."
1097             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1098             ." is irrelevant (all devices are plugged into a xhci controller).",
1099         default => 0,
1100     },
1101 };
1102
1103 my $usbdesc = {
1104     optional => 1,
1105     type => 'string', format => $usb_fmt,
1106     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1107         ." l26 or windows > 7, n can be up to 14).",
1108 };
1109 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1110
1111 my $serialdesc = {
1112         optional => 1,
1113         type => 'string',
1114         pattern => '(/dev/.+|socket)',
1115         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1116         verbose_description =>  <<EODESCR,
1117 Create a serial device inside the VM (n is 0 to 3), and pass through a
1118 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1119 host side (use 'qm terminal' to open a terminal connection).
1120
1121 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1122 use with special care.
1123
1124 CAUTION: Experimental! User reported problems with this option.
1125 EODESCR
1126 };
1127
1128 my $paralleldesc= {
1129         optional => 1,
1130         type => 'string',
1131         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1132         description =>  "Map host parallel devices (n is 0 to 2).",
1133         verbose_description =>  <<EODESCR,
1134 Map host parallel devices (n is 0 to 2).
1135
1136 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1137 machines - use with special care.
1138
1139 CAUTION: Experimental! User reported problems with this option.
1140 EODESCR
1141 };
1142
1143 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1144     $confdesc->{"parallel$i"} = $paralleldesc;
1145 }
1146
1147 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1148     $confdesc->{"serial$i"} = $serialdesc;
1149 }
1150
1151 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1152     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1153 }
1154
1155 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1156     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1157 }
1158
1159 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1160     $confdesc->{"usb$i"} = $usbdesc;
1161 }
1162
1163 my $boot_fmt = {
1164     legacy => {
1165         optional => 1,
1166         default_key => 1,
1167         type => 'string',
1168         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1169                      . " Deprecated, use 'order=' instead.",
1170         pattern => '[acdn]{1,4}',
1171         format_description => "[acdn]{1,4}",
1172
1173         # note: this is also the fallback if boot: is not given at all
1174         default => 'cdn',
1175     },
1176     order => {
1177         optional => 1,
1178         type => 'string',
1179         format => 'pve-qm-bootdev-list',
1180         format_description => "device[;device...]",
1181         description => <<EODESC,
1182 The guest will attempt to boot from devices in the order they appear here.
1183
1184 Disks, optical drives and passed-through storage USB devices will be directly
1185 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1186 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1187
1188 Note that only devices in this list will be marked as bootable and thus loaded
1189 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1190 (e.g. software-raid), you need to specify all of them here.
1191
1192 Overrides the deprecated 'legacy=[acdn]*' value when given.
1193 EODESC
1194     },
1195 };
1196 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1197
1198 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1199 sub verify_bootdev {
1200     my ($dev, $noerr) = @_;
1201
1202     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1203     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1204
1205     my $check = sub {
1206         my ($base) = @_;
1207         return 0 if $dev !~ m/^$base\d+$/;
1208         return 0 if !$confdesc->{$dev};
1209         return 1;
1210     };
1211
1212     return $dev if $check->("net");
1213     return $dev if $check->("usb");
1214     return $dev if $check->("hostpci");
1215
1216     return if $noerr;
1217     die "invalid boot device '$dev'\n";
1218 }
1219
1220 sub print_bootorder {
1221     my ($devs) = @_;
1222     return "" if !@$devs;
1223     my $data = { order => join(';', @$devs) };
1224     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1225 }
1226
1227 my $kvm_api_version = 0;
1228
1229 sub kvm_version {
1230     return $kvm_api_version if $kvm_api_version;
1231
1232     open my $fh, '<', '/dev/kvm' or return;
1233
1234     # 0xae00 => KVM_GET_API_VERSION
1235     $kvm_api_version = ioctl($fh, 0xae00, 0);
1236     close($fh);
1237
1238     return $kvm_api_version;
1239 }
1240
1241 my $kvm_user_version = {};
1242 my $kvm_mtime = {};
1243
1244 sub kvm_user_version {
1245     my ($binary) = @_;
1246
1247     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1248     my $st = stat($binary);
1249
1250     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1251     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1252         $cachedmtime == $st->mtime;
1253
1254     $kvm_user_version->{$binary} = 'unknown';
1255     $kvm_mtime->{$binary} = $st->mtime;
1256
1257     my $code = sub {
1258         my $line = shift;
1259         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1260             $kvm_user_version->{$binary} = $2;
1261         }
1262     };
1263
1264     eval { run_command([$binary, '--version'], outfunc => $code); };
1265     warn $@ if $@;
1266
1267     return $kvm_user_version->{$binary};
1268
1269 }
1270 my sub extract_version {
1271     my ($machine_type, $version) = @_;
1272     $version = kvm_user_version() if !defined($version);
1273     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1274 }
1275
1276 sub kernel_has_vhost_net {
1277     return -c '/dev/vhost-net';
1278 }
1279
1280 sub option_exists {
1281     my $key = shift;
1282     return defined($confdesc->{$key});
1283 }
1284
1285 my $cdrom_path;
1286 sub get_cdrom_path {
1287
1288     return $cdrom_path if defined($cdrom_path);
1289
1290     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1291
1292     if (!defined($cdrom_path)) {
1293         log_warn("no physical CD-ROM available, ignoring");
1294         $cdrom_path = '';
1295     }
1296
1297     return $cdrom_path;
1298 }
1299
1300 sub get_iso_path {
1301     my ($storecfg, $vmid, $cdrom) = @_;
1302
1303     if ($cdrom eq 'cdrom') {
1304         return get_cdrom_path();
1305     } elsif ($cdrom eq 'none') {
1306         return '';
1307     } elsif ($cdrom =~ m|^/|) {
1308         return $cdrom;
1309     } else {
1310         return PVE::Storage::path($storecfg, $cdrom);
1311     }
1312 }
1313
1314 # try to convert old style file names to volume IDs
1315 sub filename_to_volume_id {
1316     my ($vmid, $file, $media) = @_;
1317
1318      if (!($file eq 'none' || $file eq 'cdrom' ||
1319           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1320
1321         return if $file =~ m|/|;
1322
1323         if ($media && $media eq 'cdrom') {
1324             $file = "local:iso/$file";
1325         } else {
1326             $file = "local:$vmid/$file";
1327         }
1328     }
1329
1330     return $file;
1331 }
1332
1333 sub verify_media_type {
1334     my ($opt, $vtype, $media) = @_;
1335
1336     return if !$media;
1337
1338     my $etype;
1339     if ($media eq 'disk') {
1340         $etype = 'images';
1341     } elsif ($media eq 'cdrom') {
1342         $etype = 'iso';
1343     } else {
1344         die "internal error";
1345     }
1346
1347     return if ($vtype eq $etype);
1348
1349     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1350 }
1351
1352 sub cleanup_drive_path {
1353     my ($opt, $storecfg, $drive) = @_;
1354
1355     # try to convert filesystem paths to volume IDs
1356
1357     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1358         ($drive->{file} !~ m|^/dev/.+|) &&
1359         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1360         ($drive->{file} !~ m/^\d+$/)) {
1361         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1362         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1363             if !$vtype;
1364         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1365         verify_media_type($opt, $vtype, $drive->{media});
1366         $drive->{file} = $volid;
1367     }
1368
1369     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1370 }
1371
1372 sub parse_hotplug_features {
1373     my ($data) = @_;
1374
1375     my $res = {};
1376
1377     return $res if $data eq '0';
1378
1379     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1380
1381     foreach my $feature (PVE::Tools::split_list($data)) {
1382         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1383             $res->{$1} = 1;
1384         } else {
1385             die "invalid hotplug feature '$feature'\n";
1386         }
1387     }
1388     return $res;
1389 }
1390
1391 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1392 sub pve_verify_hotplug_features {
1393     my ($value, $noerr) = @_;
1394
1395     return $value if parse_hotplug_features($value);
1396
1397     return if $noerr;
1398
1399     die "unable to parse hotplug option\n";
1400 }
1401
1402 sub scsi_inquiry {
1403     my($fh, $noerr) = @_;
1404
1405     my $SG_IO = 0x2285;
1406     my $SG_GET_VERSION_NUM = 0x2282;
1407
1408     my $versionbuf = "\x00" x 8;
1409     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1410     if (!$ret) {
1411         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1412         return;
1413     }
1414     my $version = unpack("I", $versionbuf);
1415     if ($version < 30000) {
1416         die "scsi generic interface too old\n"  if !$noerr;
1417         return;
1418     }
1419
1420     my $buf = "\x00" x 36;
1421     my $sensebuf = "\x00" x 8;
1422     my $cmd = pack("C x3 C x1", 0x12, 36);
1423
1424     # see /usr/include/scsi/sg.h
1425     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1426
1427     my $packet = pack(
1428         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1429     );
1430
1431     $ret = ioctl($fh, $SG_IO, $packet);
1432     if (!$ret) {
1433         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1434         return;
1435     }
1436
1437     my @res = unpack($sg_io_hdr_t, $packet);
1438     if ($res[17] || $res[18]) {
1439         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1440         return;
1441     }
1442
1443     my $res = {};
1444     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1445
1446     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1447     $res->{type} &= 0x1F;
1448
1449     return $res;
1450 }
1451
1452 sub path_is_scsi {
1453     my ($path) = @_;
1454
1455     my $fh = IO::File->new("+<$path") || return;
1456     my $res = scsi_inquiry($fh, 1);
1457     close($fh);
1458
1459     return $res;
1460 }
1461
1462 sub print_tabletdevice_full {
1463     my ($conf, $arch) = @_;
1464
1465     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1466
1467     # we use uhci for old VMs because tablet driver was buggy in older qemu
1468     my $usbbus;
1469     if ($q35 || $arch eq 'aarch64') {
1470         $usbbus = 'ehci';
1471     } else {
1472         $usbbus = 'uhci';
1473     }
1474
1475     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1476 }
1477
1478 sub print_keyboarddevice_full {
1479     my ($conf, $arch) = @_;
1480
1481     return if $arch ne 'aarch64';
1482
1483     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1484 }
1485
1486 my sub get_drive_id {
1487     my ($drive) = @_;
1488     return "$drive->{interface}$drive->{index}";
1489 }
1490
1491 sub print_drivedevice_full {
1492     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1493
1494     my $device = '';
1495     my $maxdev = 0;
1496
1497     my $drive_id = get_drive_id($drive);
1498     if ($drive->{interface} eq 'virtio') {
1499         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1500         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1501         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1502     } elsif ($drive->{interface} eq 'scsi') {
1503
1504         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1505         my $unit = $drive->{index} % $maxdev;
1506         my $devicetype = 'hd';
1507         my $path = '';
1508         if (drive_is_cdrom($drive)) {
1509             $devicetype = 'cd';
1510         } else {
1511             if ($drive->{file} =~ m|^/|) {
1512                 $path = $drive->{file};
1513                 if (my $info = path_is_scsi($path)) {
1514                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1515                         $devicetype = 'block';
1516                     } elsif ($info->{type} == 1) { # tape
1517                         $devicetype = 'generic';
1518                     }
1519                 }
1520             } else {
1521                  $path = PVE::Storage::path($storecfg, $drive->{file});
1522             }
1523
1524             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1525             my $version = extract_version($machine_type, kvm_user_version());
1526             if ($path =~ m/^iscsi\:\/\// &&
1527                !min_version($version, 4, 1)) {
1528                 $devicetype = 'generic';
1529             }
1530         }
1531
1532         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1533             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1534         } else {
1535             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1536                 .",lun=$drive->{index}";
1537         }
1538         $device .= ",drive=drive-$drive_id,id=$drive_id";
1539
1540         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1541             $device .= ",rotation_rate=1";
1542         }
1543         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1544
1545     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1546         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1547         my $controller = int($drive->{index} / $maxdev);
1548         my $unit = $drive->{index} % $maxdev;
1549         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1550
1551         $device = "ide-$devicetype";
1552         if ($drive->{interface} eq 'ide') {
1553             $device .= ",bus=ide.$controller,unit=$unit";
1554         } else {
1555             $device .= ",bus=ahci$controller.$unit";
1556         }
1557         $device .= ",drive=drive-$drive_id,id=$drive_id";
1558
1559         if ($devicetype eq 'hd') {
1560             if (my $model = $drive->{model}) {
1561                 $model = URI::Escape::uri_unescape($model);
1562                 $device .= ",model=$model";
1563             }
1564             if ($drive->{ssd}) {
1565                 $device .= ",rotation_rate=1";
1566             }
1567         }
1568         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1569     } elsif ($drive->{interface} eq 'usb') {
1570         die "implement me";
1571         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1572     } else {
1573         die "unsupported interface type";
1574     }
1575
1576     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1577
1578     if (my $serial = $drive->{serial}) {
1579         $serial = URI::Escape::uri_unescape($serial);
1580         $device .= ",serial=$serial";
1581     }
1582
1583
1584     return $device;
1585 }
1586
1587 sub get_initiator_name {
1588     my $initiator;
1589
1590     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1591     while (defined(my $line = <$fh>)) {
1592         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1593         $initiator = $1;
1594         last;
1595     }
1596     $fh->close();
1597
1598     return $initiator;
1599 }
1600
1601 my sub storage_allows_io_uring_default {
1602     my ($scfg, $cache_direct) = @_;
1603
1604     # io_uring with cache mode writeback or writethrough on krbd will hang...
1605     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1606
1607     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1608     # sometimes, just plain disable...
1609     return if $scfg && $scfg->{type} eq 'lvm';
1610
1611     # io_uring causes problems when used with CIFS since kernel 5.15
1612     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1613     return if $scfg && $scfg->{type} eq 'cifs';
1614
1615     return 1;
1616 }
1617
1618 my sub drive_uses_cache_direct {
1619     my ($drive, $scfg) = @_;
1620
1621     my $cache_direct = 0;
1622
1623     if (my $cache = $drive->{cache}) {
1624         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1625     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1626         $cache_direct = 1;
1627     }
1628
1629     return $cache_direct;
1630 }
1631
1632 sub print_drive_commandline_full {
1633     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1634
1635     my $path;
1636     my $volid = $drive->{file};
1637     my $format = $drive->{format};
1638     my $drive_id = get_drive_id($drive);
1639
1640     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1641     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1642
1643     if (drive_is_cdrom($drive)) {
1644         $path = get_iso_path($storecfg, $vmid, $volid);
1645         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1646     } else {
1647         if ($storeid) {
1648             $path = PVE::Storage::path($storecfg, $volid);
1649             $format //= qemu_img_format($scfg, $volname);
1650         } else {
1651             $path = $volid;
1652             $format //= "raw";
1653         }
1654    }
1655
1656    my $is_rbd = $path =~ m/^rbd:/;
1657
1658     my $opts = '';
1659     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1660     foreach my $o (@qemu_drive_options) {
1661         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1662     }
1663
1664     # snapshot only accepts on|off
1665     if (defined($drive->{snapshot})) {
1666         my $v = $drive->{snapshot} ? 'on' : 'off';
1667         $opts .= ",snapshot=$v";
1668     }
1669
1670     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1671         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1672     }
1673
1674     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1675         my ($dir, $qmpname) = @$type;
1676         if (my $v = $drive->{"mbps$dir"}) {
1677             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1678         }
1679         if (my $v = $drive->{"mbps${dir}_max"}) {
1680             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1681         }
1682         if (my $v = $drive->{"bps${dir}_max_length"}) {
1683             $opts .= ",throttling.bps$qmpname-max-length=$v";
1684         }
1685         if (my $v = $drive->{"iops${dir}"}) {
1686             $opts .= ",throttling.iops$qmpname=$v";
1687         }
1688         if (my $v = $drive->{"iops${dir}_max"}) {
1689             $opts .= ",throttling.iops$qmpname-max=$v";
1690         }
1691         if (my $v = $drive->{"iops${dir}_max_length"}) {
1692             $opts .= ",throttling.iops$qmpname-max-length=$v";
1693         }
1694     }
1695
1696     if ($pbs_name) {
1697         $format = "rbd" if $is_rbd;
1698         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1699             if !$format;
1700         $opts .= ",format=alloc-track,file.driver=$format";
1701     } elsif ($format) {
1702         $opts .= ",format=$format";
1703     }
1704
1705     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1706
1707     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1708
1709     if (!$drive->{aio}) {
1710         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1711             # io_uring supports all cache modes
1712             $opts .= ",aio=io_uring";
1713         } else {
1714             # aio native works only with O_DIRECT
1715             if($cache_direct) {
1716                 $opts .= ",aio=native";
1717             } else {
1718                 $opts .= ",aio=threads";
1719             }
1720         }
1721     }
1722
1723     if (!drive_is_cdrom($drive)) {
1724         my $detectzeroes;
1725         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1726             $detectzeroes = 'off';
1727         } elsif ($drive->{discard}) {
1728             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1729         } else {
1730             # This used to be our default with discard not being specified:
1731             $detectzeroes = 'on';
1732         }
1733
1734         # note: 'detect-zeroes' works per blockdev and we want it to persist
1735         # after the alloc-track is removed, so put it on 'file' directly
1736         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1737         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1738     }
1739
1740     if ($pbs_name) {
1741         $opts .= ",backing=$pbs_name";
1742         $opts .= ",auto-remove=on";
1743     }
1744
1745     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1746     my $file_param = "file";
1747     if ($pbs_name) {
1748         # non-rbd drivers require the underlying file to be a seperate block
1749         # node, so add a second .file indirection
1750         $file_param .= ".file" if !$is_rbd;
1751         $file_param .= ".filename";
1752     }
1753     my $pathinfo = $path ? "$file_param=$path," : '';
1754
1755     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1756 }
1757
1758 sub print_pbs_blockdev {
1759     my ($pbs_conf, $pbs_name) = @_;
1760     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1761     $blockdev .= ",repository=$pbs_conf->{repository}";
1762     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1763     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1764     $blockdev .= ",archive=$pbs_conf->{archive}";
1765     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1766     return $blockdev;
1767 }
1768
1769 sub print_netdevice_full {
1770     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1771
1772     my $device = $net->{model};
1773     if ($net->{model} eq 'virtio') {
1774          $device = 'virtio-net-pci';
1775      };
1776
1777     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1778     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1779     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1780         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1781         # and out of each queue plus one config interrupt and control vector queue
1782         my $vectors = $net->{queues} * 2 + 2;
1783         $tmpstr .= ",vectors=$vectors,mq=on";
1784         if (min_version($machine_version, 7, 1)) {
1785             $tmpstr .= ",packed=on";
1786         }
1787     }
1788
1789     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1790         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1791     }
1792
1793     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1794
1795     if (my $mtu = $net->{mtu}) {
1796         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1797             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1798             if ($mtu == 1) {
1799                  $mtu = $bridge_mtu;
1800             } elsif ($mtu < 576) {
1801                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1802             } elsif ($mtu > $bridge_mtu) {
1803                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1804             }
1805             $tmpstr .= ",host_mtu=$mtu";
1806         } else {
1807             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1808         }
1809     }
1810
1811     if ($use_old_bios_files) {
1812         my $romfile;
1813         if ($device eq 'virtio-net-pci') {
1814             $romfile = 'pxe-virtio.rom';
1815         } elsif ($device eq 'e1000') {
1816             $romfile = 'pxe-e1000.rom';
1817         } elsif ($device eq 'e1000e') {
1818             $romfile = 'pxe-e1000e.rom';
1819         } elsif ($device eq 'ne2k') {
1820             $romfile = 'pxe-ne2k_pci.rom';
1821         } elsif ($device eq 'pcnet') {
1822             $romfile = 'pxe-pcnet.rom';
1823         } elsif ($device eq 'rtl8139') {
1824             $romfile = 'pxe-rtl8139.rom';
1825         }
1826         $tmpstr .= ",romfile=$romfile" if $romfile;
1827     }
1828
1829     return $tmpstr;
1830 }
1831
1832 sub print_netdev_full {
1833     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1834
1835     my $i = '';
1836     if ($netid =~ m/^net(\d+)$/) {
1837         $i = int($1);
1838     }
1839
1840     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1841
1842     my $ifname = "tap${vmid}i$i";
1843
1844     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1845     die "interface name '$ifname' is too long (max 15 character)\n"
1846         if length($ifname) >= 16;
1847
1848     my $vhostparam = '';
1849     if (is_native($arch)) {
1850         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1851     }
1852
1853     my $vmname = $conf->{name} || "vm$vmid";
1854
1855     my $netdev = "";
1856     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1857
1858     if ($net->{bridge}) {
1859         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1860             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1861     } else {
1862         $netdev = "type=user,id=$netid,hostname=$vmname";
1863     }
1864
1865     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1866
1867     return $netdev;
1868 }
1869
1870 my $vga_map = {
1871     'cirrus' => 'cirrus-vga',
1872     'std' => 'VGA',
1873     'vmware' => 'vmware-svga',
1874     'virtio' => 'virtio-vga',
1875     'virtio-gl' => 'virtio-vga-gl',
1876 };
1877
1878 sub print_vga_device {
1879     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1880
1881     my $type = $vga_map->{$vga->{type}};
1882     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1883         $type = 'virtio-gpu';
1884     }
1885     my $vgamem_mb = $vga->{memory};
1886
1887     my $max_outputs = '';
1888     if ($qxlnum) {
1889         $type = $id ? 'qxl' : 'qxl-vga';
1890
1891         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1892             # set max outputs so linux can have up to 4 qxl displays with one device
1893             if (min_version($machine_version, 4, 1)) {
1894                 $max_outputs = ",max_outputs=4";
1895             }
1896         }
1897     }
1898
1899     die "no devicetype for $vga->{type}\n" if !$type;
1900
1901     my $memory = "";
1902     if ($vgamem_mb) {
1903         if ($vga->{type} =~ /^virtio/) {
1904             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1905             $memory = ",max_hostmem=$bytes";
1906         } elsif ($qxlnum) {
1907             # from https://www.spice-space.org/multiple-monitors.html
1908             $memory = ",vgamem_mb=$vga->{memory}";
1909             my $ram = $vgamem_mb * 4;
1910             my $vram = $vgamem_mb * 2;
1911             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1912         } else {
1913             $memory = ",vgamem_mb=$vga->{memory}";
1914         }
1915     } elsif ($qxlnum && $id) {
1916         $memory = ",ram_size=67108864,vram_size=33554432";
1917     }
1918
1919     my $edidoff = "";
1920     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1921         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1922     }
1923
1924     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1925     my $vgaid = "vga" . ($id // '');
1926     my $pciaddr;
1927     if ($q35 && $vgaid eq 'vga') {
1928         # the first display uses pcie.0 bus on q35 machines
1929         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1930     } else {
1931         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1932     }
1933
1934     if ($vga->{type} eq 'virtio-gl') {
1935         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1936         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1937             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1938
1939         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1940             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1941     }
1942
1943     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1944 }
1945
1946 sub parse_number_sets {
1947     my ($set) = @_;
1948     my $res = [];
1949     foreach my $part (split(/;/, $set)) {
1950         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1951             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1952             push @$res, [ $1, $2 ];
1953         } else {
1954             die "invalid range: $part\n";
1955         }
1956     }
1957     return $res;
1958 }
1959
1960 sub parse_numa {
1961     my ($data) = @_;
1962
1963     my $res = parse_property_string($numa_fmt, $data);
1964     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1965     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1966     return $res;
1967 }
1968
1969 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1970 sub parse_net {
1971     my ($data, $disable_mac_autogen) = @_;
1972
1973     my $res = eval { parse_property_string($net_fmt, $data) };
1974     if ($@) {
1975         warn $@;
1976         return;
1977     }
1978     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1979         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1980         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1981     }
1982     return $res;
1983 }
1984
1985 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1986 sub parse_ipconfig {
1987     my ($data) = @_;
1988
1989     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1990     if ($@) {
1991         warn $@;
1992         return;
1993     }
1994
1995     if ($res->{gw} && !$res->{ip}) {
1996         warn 'gateway specified without specifying an IP address';
1997         return;
1998     }
1999     if ($res->{gw6} && !$res->{ip6}) {
2000         warn 'IPv6 gateway specified without specifying an IPv6 address';
2001         return;
2002     }
2003     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2004         warn 'gateway specified together with DHCP';
2005         return;
2006     }
2007     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2008         # gw6 + auto/dhcp
2009         warn "IPv6 gateway specified together with $res->{ip6} address";
2010         return;
2011     }
2012
2013     if (!$res->{ip} && !$res->{ip6}) {
2014         return { ip => 'dhcp', ip6 => 'dhcp' };
2015     }
2016
2017     return $res;
2018 }
2019
2020 sub print_net {
2021     my $net = shift;
2022
2023     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2024 }
2025
2026 sub add_random_macs {
2027     my ($settings) = @_;
2028
2029     foreach my $opt (keys %$settings) {
2030         next if $opt !~ m/^net(\d+)$/;
2031         my $net = parse_net($settings->{$opt});
2032         next if !$net;
2033         $settings->{$opt} = print_net($net);
2034     }
2035 }
2036
2037 sub vm_is_volid_owner {
2038     my ($storecfg, $vmid, $volid) = @_;
2039
2040     if ($volid !~  m|^/|) {
2041         my ($path, $owner);
2042         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2043         if ($owner && ($owner == $vmid)) {
2044             return 1;
2045         }
2046     }
2047
2048     return;
2049 }
2050
2051 sub vmconfig_register_unused_drive {
2052     my ($storecfg, $vmid, $conf, $drive) = @_;
2053
2054     if (drive_is_cloudinit($drive)) {
2055         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2056         warn $@ if $@;
2057         delete $conf->{cloudinit};
2058     } elsif (!drive_is_cdrom($drive)) {
2059         my $volid = $drive->{file};
2060         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2061             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2062         }
2063     }
2064 }
2065
2066 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2067 my $smbios1_fmt = {
2068     uuid => {
2069         type => 'string',
2070         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2071         format_description => 'UUID',
2072         description => "Set SMBIOS1 UUID.",
2073         optional => 1,
2074     },
2075     version => {
2076         type => 'string',
2077         pattern => '[A-Za-z0-9+\/]+={0,2}',
2078         format_description => 'Base64 encoded string',
2079         description => "Set SMBIOS1 version.",
2080         optional => 1,
2081     },
2082     serial => {
2083         type => 'string',
2084         pattern => '[A-Za-z0-9+\/]+={0,2}',
2085         format_description => 'Base64 encoded string',
2086         description => "Set SMBIOS1 serial number.",
2087         optional => 1,
2088     },
2089     manufacturer => {
2090         type => 'string',
2091         pattern => '[A-Za-z0-9+\/]+={0,2}',
2092         format_description => 'Base64 encoded string',
2093         description => "Set SMBIOS1 manufacturer.",
2094         optional => 1,
2095     },
2096     product => {
2097         type => 'string',
2098         pattern => '[A-Za-z0-9+\/]+={0,2}',
2099         format_description => 'Base64 encoded string',
2100         description => "Set SMBIOS1 product ID.",
2101         optional => 1,
2102     },
2103     sku => {
2104         type => 'string',
2105         pattern => '[A-Za-z0-9+\/]+={0,2}',
2106         format_description => 'Base64 encoded string',
2107         description => "Set SMBIOS1 SKU string.",
2108         optional => 1,
2109     },
2110     family => {
2111         type => 'string',
2112         pattern => '[A-Za-z0-9+\/]+={0,2}',
2113         format_description => 'Base64 encoded string',
2114         description => "Set SMBIOS1 family string.",
2115         optional => 1,
2116     },
2117     base64 => {
2118         type => 'boolean',
2119         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2120         optional => 1,
2121     },
2122 };
2123
2124 sub parse_smbios1 {
2125     my ($data) = @_;
2126
2127     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2128     warn $@ if $@;
2129     return $res;
2130 }
2131
2132 sub print_smbios1 {
2133     my ($smbios1) = @_;
2134     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2135 }
2136
2137 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2138
2139 sub parse_watchdog {
2140     my ($value) = @_;
2141
2142     return if !$value;
2143
2144     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2145     warn $@ if $@;
2146     return $res;
2147 }
2148
2149 sub parse_guest_agent {
2150     my ($conf) = @_;
2151
2152     return {} if !defined($conf->{agent});
2153
2154     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2155     warn $@ if $@;
2156
2157     # if the agent is disabled ignore the other potentially set properties
2158     return {} if !$res->{enabled};
2159     return $res;
2160 }
2161
2162 sub get_qga_key {
2163     my ($conf, $key) = @_;
2164     return undef if !defined($conf->{agent});
2165
2166     my $agent = parse_guest_agent($conf);
2167     return $agent->{$key};
2168 }
2169
2170 sub parse_vga {
2171     my ($value) = @_;
2172
2173     return {} if !$value;
2174     my $res = eval { parse_property_string($vga_fmt, $value) };
2175     warn $@ if $@;
2176     return $res;
2177 }
2178
2179 sub parse_rng {
2180     my ($value) = @_;
2181
2182     return if !$value;
2183
2184     my $res = eval { parse_property_string($rng_fmt, $value) };
2185     warn $@ if $@;
2186     return $res;
2187 }
2188
2189 sub parse_meta_info {
2190     my ($value) = @_;
2191
2192     return if !$value;
2193
2194     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2195     warn $@ if $@;
2196     return $res;
2197 }
2198
2199 sub new_meta_info_string {
2200     my () = @_; # for now do not allow to override any value
2201
2202     return PVE::JSONSchema::print_property_string(
2203         {
2204             'creation-qemu' => kvm_user_version(),
2205             ctime => "". int(time()),
2206         },
2207         $meta_info_fmt
2208     );
2209 }
2210
2211 sub qemu_created_version_fixups {
2212     my ($conf, $forcemachine, $kvmver) = @_;
2213
2214     my $meta = parse_meta_info($conf->{meta}) // {};
2215     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2216
2217     # check if we need to apply some handling for VMs that always use the latest machine version but
2218     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2219     # would be required (we do not want to pin machine version for non-windows OS type)
2220     if (
2221         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2222         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2223         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2224         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2225     ) {
2226         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2227         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2228             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2229             # and thus with the predictable interface naming of systemd
2230             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2231         }
2232     }
2233     return;
2234 }
2235
2236 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2237 sub verify_usb_device {
2238     my ($value, $noerr) = @_;
2239
2240     return $value if parse_usb_device($value);
2241
2242     return if $noerr;
2243
2244     die "unable to parse usb device\n";
2245 }
2246
2247 # add JSON properties for create and set function
2248 sub json_config_properties {
2249     my ($prop, $with_disk_alloc) = @_;
2250
2251     my $skip_json_config_opts = {
2252         parent => 1,
2253         snaptime => 1,
2254         vmstate => 1,
2255         runningmachine => 1,
2256         runningcpu => 1,
2257         meta => 1,
2258     };
2259
2260     foreach my $opt (keys %$confdesc) {
2261         next if $skip_json_config_opts->{$opt};
2262
2263         if ($with_disk_alloc && is_valid_drivename($opt)) {
2264             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2265         } else {
2266             $prop->{$opt} = $confdesc->{$opt};
2267         }
2268     }
2269
2270     return $prop;
2271 }
2272
2273 # Properties that we can read from an OVF file
2274 sub json_ovf_properties {
2275     my $prop = {};
2276
2277     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2278         $prop->{$device} = {
2279             type => 'string',
2280             format => 'pve-volume-id-or-absolute-path',
2281             description => "Disk image that gets imported to $device",
2282             optional => 1,
2283         };
2284     }
2285
2286     $prop->{cores} = {
2287         type => 'integer',
2288         description => "The number of CPU cores.",
2289         optional => 1,
2290     };
2291     $prop->{memory} = {
2292         type => 'integer',
2293         description => "Amount of RAM for the VM in MB.",
2294         optional => 1,
2295     };
2296     $prop->{name} = {
2297         type => 'string',
2298         description => "Name of the VM.",
2299         optional => 1,
2300     };
2301
2302     return $prop;
2303 }
2304
2305 # return copy of $confdesc_cloudinit to generate documentation
2306 sub cloudinit_config_properties {
2307
2308     return dclone($confdesc_cloudinit);
2309 }
2310
2311 sub cloudinit_pending_properties {
2312     my $p = {
2313         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2314         name => 1,
2315     };
2316     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2317     return $p;
2318 }
2319
2320 sub check_type {
2321     my ($key, $value) = @_;
2322
2323     die "unknown setting '$key'\n" if !$confdesc->{$key};
2324
2325     my $type = $confdesc->{$key}->{type};
2326
2327     if (!defined($value)) {
2328         die "got undefined value\n";
2329     }
2330
2331     if ($value =~ m/[\n\r]/) {
2332         die "property contains a line feed\n";
2333     }
2334
2335     if ($type eq 'boolean') {
2336         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2337         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2338         die "type check ('boolean') failed - got '$value'\n";
2339     } elsif ($type eq 'integer') {
2340         return int($1) if $value =~ m/^(\d+)$/;
2341         die "type check ('integer') failed - got '$value'\n";
2342     } elsif ($type eq 'number') {
2343         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2344         die "type check ('number') failed - got '$value'\n";
2345     } elsif ($type eq 'string') {
2346         if (my $fmt = $confdesc->{$key}->{format}) {
2347             PVE::JSONSchema::check_format($fmt, $value);
2348             return $value;
2349         }
2350         $value =~ s/^\"(.*)\"$/$1/;
2351         return $value;
2352     } else {
2353         die "internal error"
2354     }
2355 }
2356
2357 sub destroy_vm {
2358     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2359
2360     my $conf = PVE::QemuConfig->load_config($vmid);
2361
2362     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2363         PVE::QemuConfig->check_lock($conf);
2364     }
2365
2366     if ($conf->{template}) {
2367         # check if any base image is still used by a linked clone
2368         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2369                 my ($ds, $drive) = @_;
2370                 return if drive_is_cdrom($drive);
2371
2372                 my $volid = $drive->{file};
2373                 return if !$volid || $volid =~ m|^/|;
2374
2375                 die "base volume '$volid' is still in use by linked cloned\n"
2376                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2377
2378         });
2379     }
2380
2381     my $volids = {};
2382     my $remove_owned_drive = sub {
2383         my ($ds, $drive) = @_;
2384         return if drive_is_cdrom($drive, 1);
2385
2386         my $volid = $drive->{file};
2387         return if !$volid || $volid =~ m|^/|;
2388         return if $volids->{$volid};
2389
2390         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2391         return if !$path || !$owner || ($owner != $vmid);
2392
2393         $volids->{$volid} = 1;
2394         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2395         warn "Could not remove disk '$volid', check manually: $@" if $@;
2396     };
2397
2398     # only remove disks owned by this VM (referenced in the config)
2399     my $include_opts = {
2400         include_unused => 1,
2401         extra_keys => ['vmstate'],
2402     };
2403     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2404
2405     for my $snap (values %{$conf->{snapshots}}) {
2406         next if !defined($snap->{vmstate});
2407         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2408         next if !defined($drive);
2409         $remove_owned_drive->('vmstate', $drive);
2410     }
2411
2412     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2413
2414     if ($purge_unreferenced) { # also remove unreferenced disk
2415         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2416         PVE::Storage::foreach_volid($vmdisks, sub {
2417             my ($volid, $sid, $volname, $d) = @_;
2418             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2419             warn $@ if $@;
2420         });
2421     }
2422
2423     if (defined $replacement_conf) {
2424         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2425     } else {
2426         PVE::QemuConfig->destroy_config($vmid);
2427     }
2428 }
2429
2430 sub parse_vm_config {
2431     my ($filename, $raw, $strict) = @_;
2432
2433     return if !defined($raw);
2434
2435     my $res = {
2436         digest => Digest::SHA::sha1_hex($raw),
2437         snapshots => {},
2438         pending => {},
2439         cloudinit => {},
2440     };
2441
2442     my $handle_error = sub {
2443         my ($msg) = @_;
2444
2445         if ($strict) {
2446             die $msg;
2447         } else {
2448             warn $msg;
2449         }
2450     };
2451
2452     $filename =~ m|/qemu-server/(\d+)\.conf$|
2453         || die "got strange filename '$filename'";
2454
2455     my $vmid = $1;
2456
2457     my $conf = $res;
2458     my $descr;
2459     my $finish_description = sub {
2460         if (defined($descr)) {
2461             $descr =~ s/\s+$//;
2462             $conf->{description} = $descr;
2463         }
2464         $descr = undef;
2465     };
2466     my $section = '';
2467
2468     my @lines = split(/\n/, $raw);
2469     foreach my $line (@lines) {
2470         next if $line =~ m/^\s*$/;
2471
2472         if ($line =~ m/^\[PENDING\]\s*$/i) {
2473             $section = 'pending';
2474             $finish_description->();
2475             $conf = $res->{$section} = {};
2476             next;
2477         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2478             $section = 'cloudinit';
2479             $finish_description->();
2480             $conf = $res->{$section} = {};
2481             next;
2482
2483         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2484             $section = $1;
2485             $finish_description->();
2486             $conf = $res->{snapshots}->{$section} = {};
2487             next;
2488         }
2489
2490         if ($line =~ m/^\#(.*)$/) {
2491             $descr = '' if !defined($descr);
2492             $descr .= PVE::Tools::decode_text($1) . "\n";
2493             next;
2494         }
2495
2496         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2497             $descr = '' if !defined($descr);
2498             $descr .= PVE::Tools::decode_text($2);
2499         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2500             $conf->{snapstate} = $1;
2501         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2502             my $key = $1;
2503             my $value = $2;
2504             $conf->{$key} = $value;
2505         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2506             my $value = $1;
2507             if ($section eq 'pending') {
2508                 $conf->{delete} = $value; # we parse this later
2509             } else {
2510                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2511             }
2512         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2513             my $key = $1;
2514             my $value = $2;
2515             if ($section eq 'cloudinit') {
2516                 # ignore validation only used for informative purpose
2517                 $conf->{$key} = $value;
2518                 next;
2519             }
2520             eval { $value = check_type($key, $value); };
2521             if ($@) {
2522                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2523             } else {
2524                 $key = 'ide2' if $key eq 'cdrom';
2525                 my $fmt = $confdesc->{$key}->{format};
2526                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2527                     my $v = parse_drive($key, $value);
2528                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2529                         $v->{file} = $volid;
2530                         $value = print_drive($v);
2531                     } else {
2532                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2533                         next;
2534                     }
2535                 }
2536
2537                 $conf->{$key} = $value;
2538             }
2539         } else {
2540             $handle_error->("vm $vmid - unable to parse config: $line\n");
2541         }
2542     }
2543
2544     $finish_description->();
2545     delete $res->{snapstate}; # just to be sure
2546
2547     return $res;
2548 }
2549
2550 sub write_vm_config {
2551     my ($filename, $conf) = @_;
2552
2553     delete $conf->{snapstate}; # just to be sure
2554
2555     if ($conf->{cdrom}) {
2556         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2557         $conf->{ide2} = $conf->{cdrom};
2558         delete $conf->{cdrom};
2559     }
2560
2561     # we do not use 'smp' any longer
2562     if ($conf->{sockets}) {
2563         delete $conf->{smp};
2564     } elsif ($conf->{smp}) {
2565         $conf->{sockets} = $conf->{smp};
2566         delete $conf->{cores};
2567         delete $conf->{smp};
2568     }
2569
2570     my $used_volids = {};
2571
2572     my $cleanup_config = sub {
2573         my ($cref, $pending, $snapname) = @_;
2574
2575         foreach my $key (keys %$cref) {
2576             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2577                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2578             my $value = $cref->{$key};
2579             if ($key eq 'delete') {
2580                 die "propertry 'delete' is only allowed in [PENDING]\n"
2581                     if !$pending;
2582                 # fixme: check syntax?
2583                 next;
2584             }
2585             eval { $value = check_type($key, $value); };
2586             die "unable to parse value of '$key' - $@" if $@;
2587
2588             $cref->{$key} = $value;
2589
2590             if (!$snapname && is_valid_drivename($key)) {
2591                 my $drive = parse_drive($key, $value);
2592                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2593             }
2594         }
2595     };
2596
2597     &$cleanup_config($conf);
2598
2599     &$cleanup_config($conf->{pending}, 1);
2600
2601     foreach my $snapname (keys %{$conf->{snapshots}}) {
2602         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2603         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2604     }
2605
2606     # remove 'unusedX' settings if we re-add a volume
2607     foreach my $key (keys %$conf) {
2608         my $value = $conf->{$key};
2609         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2610             delete $conf->{$key};
2611         }
2612     }
2613
2614     my $generate_raw_config = sub {
2615         my ($conf, $pending) = @_;
2616
2617         my $raw = '';
2618
2619         # add description as comment to top of file
2620         if (defined(my $descr = $conf->{description})) {
2621             if ($descr) {
2622                 foreach my $cl (split(/\n/, $descr)) {
2623                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2624                 }
2625             } else {
2626                 $raw .= "#\n" if $pending;
2627             }
2628         }
2629
2630         foreach my $key (sort keys %$conf) {
2631             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2632             $raw .= "$key: $conf->{$key}\n";
2633         }
2634         return $raw;
2635     };
2636
2637     my $raw = &$generate_raw_config($conf);
2638
2639     if (scalar(keys %{$conf->{pending}})){
2640         $raw .= "\n[PENDING]\n";
2641         $raw .= &$generate_raw_config($conf->{pending}, 1);
2642     }
2643
2644     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2645         $raw .= "\n[special:cloudinit]\n";
2646         $raw .= &$generate_raw_config($conf->{cloudinit});
2647     }
2648
2649     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2650         $raw .= "\n[$snapname]\n";
2651         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2652     }
2653
2654     return $raw;
2655 }
2656
2657 sub load_defaults {
2658
2659     my $res = {};
2660
2661     # we use static defaults from our JSON schema configuration
2662     foreach my $key (keys %$confdesc) {
2663         if (defined(my $default = $confdesc->{$key}->{default})) {
2664             $res->{$key} = $default;
2665         }
2666     }
2667
2668     return $res;
2669 }
2670
2671 sub config_list {
2672     my $vmlist = PVE::Cluster::get_vmlist();
2673     my $res = {};
2674     return $res if !$vmlist || !$vmlist->{ids};
2675     my $ids = $vmlist->{ids};
2676     my $nodename = nodename();
2677
2678     foreach my $vmid (keys %$ids) {
2679         my $d = $ids->{$vmid};
2680         next if !$d->{node} || $d->{node} ne $nodename;
2681         next if !$d->{type} || $d->{type} ne 'qemu';
2682         $res->{$vmid}->{exists} = 1;
2683     }
2684     return $res;
2685 }
2686
2687 # test if VM uses local resources (to prevent migration)
2688 sub check_local_resources {
2689     my ($conf, $noerr) = @_;
2690
2691     my @loc_res = ();
2692
2693     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2694     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2695
2696     push @loc_res, "ivshmem" if $conf->{ivshmem};
2697
2698     foreach my $k (keys %$conf) {
2699         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2700         # sockets are safe: they will recreated be on the target side post-migrate
2701         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2702         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2703     }
2704
2705     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2706
2707     return \@loc_res;
2708 }
2709
2710 # check if used storages are available on all nodes (use by migrate)
2711 sub check_storage_availability {
2712     my ($storecfg, $conf, $node) = @_;
2713
2714     PVE::QemuConfig->foreach_volume($conf, sub {
2715         my ($ds, $drive) = @_;
2716
2717         my $volid = $drive->{file};
2718         return if !$volid;
2719
2720         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2721         return if !$sid;
2722
2723         # check if storage is available on both nodes
2724         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2725         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2726
2727         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2728
2729         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2730             if !$scfg->{content}->{$vtype};
2731    });
2732 }
2733
2734 # list nodes where all VM images are available (used by has_feature API)
2735 sub shared_nodes {
2736     my ($conf, $storecfg) = @_;
2737
2738     my $nodelist = PVE::Cluster::get_nodelist();
2739     my $nodehash = { map { $_ => 1 } @$nodelist };
2740     my $nodename = nodename();
2741
2742     PVE::QemuConfig->foreach_volume($conf, sub {
2743         my ($ds, $drive) = @_;
2744
2745         my $volid = $drive->{file};
2746         return if !$volid;
2747
2748         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2749         if ($storeid) {
2750             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2751             if ($scfg->{disable}) {
2752                 $nodehash = {};
2753             } elsif (my $avail = $scfg->{nodes}) {
2754                 foreach my $node (keys %$nodehash) {
2755                     delete $nodehash->{$node} if !$avail->{$node};
2756                 }
2757             } elsif (!$scfg->{shared}) {
2758                 foreach my $node (keys %$nodehash) {
2759                     delete $nodehash->{$node} if $node ne $nodename
2760                 }
2761             }
2762         }
2763     });
2764
2765     return $nodehash
2766 }
2767
2768 sub check_local_storage_availability {
2769     my ($conf, $storecfg) = @_;
2770
2771     my $nodelist = PVE::Cluster::get_nodelist();
2772     my $nodehash = { map { $_ => {} } @$nodelist };
2773
2774     PVE::QemuConfig->foreach_volume($conf, sub {
2775         my ($ds, $drive) = @_;
2776
2777         my $volid = $drive->{file};
2778         return if !$volid;
2779
2780         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2781         if ($storeid) {
2782             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2783
2784             if ($scfg->{disable}) {
2785                 foreach my $node (keys %$nodehash) {
2786                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2787                 }
2788             } elsif (my $avail = $scfg->{nodes}) {
2789                 foreach my $node (keys %$nodehash) {
2790                     if (!$avail->{$node}) {
2791                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2792                     }
2793                 }
2794             }
2795         }
2796     });
2797
2798     foreach my $node (values %$nodehash) {
2799         if (my $unavail = $node->{unavailable_storages}) {
2800             $node->{unavailable_storages} = [ sort keys %$unavail ];
2801         }
2802     }
2803
2804     return $nodehash
2805 }
2806
2807 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2808 sub check_running {
2809     my ($vmid, $nocheck, $node) = @_;
2810
2811     # $nocheck is set when called during a migration, in which case the config
2812     # file might still or already reside on the *other* node
2813     # - because rename has already happened, and current node is source
2814     # - because rename hasn't happened yet, and current node is target
2815     # - because rename has happened, current node is target, but hasn't yet
2816     # processed it yet
2817     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2818     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2819 }
2820
2821 sub vzlist {
2822
2823     my $vzlist = config_list();
2824
2825     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2826
2827     while (defined(my $de = $fd->read)) {
2828         next if $de !~ m/^(\d+)\.pid$/;
2829         my $vmid = $1;
2830         next if !defined($vzlist->{$vmid});
2831         if (my $pid = check_running($vmid)) {
2832             $vzlist->{$vmid}->{pid} = $pid;
2833         }
2834     }
2835
2836     return $vzlist;
2837 }
2838
2839 our $vmstatus_return_properties = {
2840     vmid => get_standard_option('pve-vmid'),
2841     status => {
2842         description => "QEMU process status.",
2843         type => 'string',
2844         enum => ['stopped', 'running'],
2845     },
2846     maxmem => {
2847         description => "Maximum memory in bytes.",
2848         type => 'integer',
2849         optional => 1,
2850         renderer => 'bytes',
2851     },
2852     maxdisk => {
2853         description => "Root disk size in bytes.",
2854         type => 'integer',
2855         optional => 1,
2856         renderer => 'bytes',
2857     },
2858     name => {
2859         description => "VM name.",
2860         type => 'string',
2861         optional => 1,
2862     },
2863     qmpstatus => {
2864         description => "QEMU QMP agent status.",
2865         type => 'string',
2866         optional => 1,
2867     },
2868     pid => {
2869         description => "PID of running qemu process.",
2870         type => 'integer',
2871         optional => 1,
2872     },
2873     uptime => {
2874         description => "Uptime.",
2875         type => 'integer',
2876         optional => 1,
2877         renderer => 'duration',
2878     },
2879     cpus => {
2880         description => "Maximum usable CPUs.",
2881         type => 'number',
2882         optional => 1,
2883     },
2884     lock => {
2885         description => "The current config lock, if any.",
2886         type => 'string',
2887         optional => 1,
2888     },
2889     tags => {
2890         description  => "The current configured tags, if any",
2891         type => 'string',
2892         optional => 1,
2893     },
2894     'running-machine' => {
2895         description => "The currently running machine type (if running).",
2896         type => 'string',
2897         optional => 1,
2898     },
2899     'running-qemu' => {
2900         description => "The currently running QEMU version (if running).",
2901         type => 'string',
2902         optional => 1,
2903     },
2904 };
2905
2906 my $last_proc_pid_stat;
2907
2908 # get VM status information
2909 # This must be fast and should not block ($full == false)
2910 # We only query KVM using QMP if $full == true (this can be slow)
2911 sub vmstatus {
2912     my ($opt_vmid, $full) = @_;
2913
2914     my $res = {};
2915
2916     my $storecfg = PVE::Storage::config();
2917
2918     my $list = vzlist();
2919     my $defaults = load_defaults();
2920
2921     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2922
2923     my $cpucount = $cpuinfo->{cpus} || 1;
2924
2925     foreach my $vmid (keys %$list) {
2926         next if $opt_vmid && ($vmid ne $opt_vmid);
2927
2928         my $conf = PVE::QemuConfig->load_config($vmid);
2929
2930         my $d = { vmid => int($vmid) };
2931         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2932
2933         # fixme: better status?
2934         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2935
2936         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2937         if (defined($size)) {
2938             $d->{disk} = 0; # no info available
2939             $d->{maxdisk} = $size;
2940         } else {
2941             $d->{disk} = 0;
2942             $d->{maxdisk} = 0;
2943         }
2944
2945         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2946             * ($conf->{cores} || $defaults->{cores});
2947         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2948         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2949
2950         $d->{name} = $conf->{name} || "VM $vmid";
2951         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2952             : $defaults->{memory}*(1024*1024);
2953
2954         if ($conf->{balloon}) {
2955             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2956             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2957                 : $defaults->{shares};
2958         }
2959
2960         $d->{uptime} = 0;
2961         $d->{cpu} = 0;
2962         $d->{mem} = 0;
2963
2964         $d->{netout} = 0;
2965         $d->{netin} = 0;
2966
2967         $d->{diskread} = 0;
2968         $d->{diskwrite} = 0;
2969
2970         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2971
2972         $d->{serial} = 1 if conf_has_serial($conf);
2973         $d->{lock} = $conf->{lock} if $conf->{lock};
2974         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2975
2976         $res->{$vmid} = $d;
2977     }
2978
2979     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2980     foreach my $dev (keys %$netdev) {
2981         next if $dev !~ m/^tap([1-9]\d*)i/;
2982         my $vmid = $1;
2983         my $d = $res->{$vmid};
2984         next if !$d;
2985
2986         $d->{netout} += $netdev->{$dev}->{receive};
2987         $d->{netin} += $netdev->{$dev}->{transmit};
2988
2989         if ($full) {
2990             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2991             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2992         }
2993
2994     }
2995
2996     my $ctime = gettimeofday;
2997
2998     foreach my $vmid (keys %$list) {
2999
3000         my $d = $res->{$vmid};
3001         my $pid = $d->{pid};
3002         next if !$pid;
3003
3004         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3005         next if !$pstat; # not running
3006
3007         my $used = $pstat->{utime} + $pstat->{stime};
3008
3009         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3010
3011         if ($pstat->{vsize}) {
3012             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3013         }
3014
3015         my $old = $last_proc_pid_stat->{$pid};
3016         if (!$old) {
3017             $last_proc_pid_stat->{$pid} = {
3018                 time => $ctime,
3019                 used => $used,
3020                 cpu => 0,
3021             };
3022             next;
3023         }
3024
3025         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3026
3027         if ($dtime > 1000) {
3028             my $dutime = $used -  $old->{used};
3029
3030             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3031             $last_proc_pid_stat->{$pid} = {
3032                 time => $ctime,
3033                 used => $used,
3034                 cpu => $d->{cpu},
3035             };
3036         } else {
3037             $d->{cpu} = $old->{cpu};
3038         }
3039     }
3040
3041     return $res if !$full;
3042
3043     my $qmpclient = PVE::QMPClient->new();
3044
3045     my $ballooncb = sub {
3046         my ($vmid, $resp) = @_;
3047
3048         my $info = $resp->{'return'};
3049         return if !$info->{max_mem};
3050
3051         my $d = $res->{$vmid};
3052
3053         # use memory assigned to VM
3054         $d->{maxmem} = $info->{max_mem};
3055         $d->{balloon} = $info->{actual};
3056
3057         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3058             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3059             $d->{freemem} = $info->{free_mem};
3060         }
3061
3062         $d->{ballooninfo} = $info;
3063     };
3064
3065     my $blockstatscb = sub {
3066         my ($vmid, $resp) = @_;
3067         my $data = $resp->{'return'} || [];
3068         my $totalrdbytes = 0;
3069         my $totalwrbytes = 0;
3070
3071         for my $blockstat (@$data) {
3072             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3073             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3074
3075             $blockstat->{device} =~ s/drive-//;
3076             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3077         }
3078         $res->{$vmid}->{diskread} = $totalrdbytes;
3079         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3080     };
3081
3082     my $machinecb = sub {
3083         my ($vmid, $resp) = @_;
3084         my $data = $resp->{'return'} || [];
3085
3086         $res->{$vmid}->{'running-machine'} =
3087             PVE::QemuServer::Machine::current_from_query_machines($data);
3088     };
3089
3090     my $versioncb = sub {
3091         my ($vmid, $resp) = @_;
3092         my $data = $resp->{'return'} // {};
3093         my $version = 'unknown';
3094
3095         if (my $v = $data->{qemu}) {
3096             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3097         }
3098
3099         $res->{$vmid}->{'running-qemu'} = $version;
3100     };
3101
3102     my $statuscb = sub {
3103         my ($vmid, $resp) = @_;
3104
3105         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3106         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3107         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3108         # this fails if ballon driver is not loaded, so this must be
3109         # the last commnand (following command are aborted if this fails).
3110         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3111
3112         my $status = 'unknown';
3113         if (!defined($status = $resp->{'return'}->{status})) {
3114             warn "unable to get VM status\n";
3115             return;
3116         }
3117
3118         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3119     };
3120
3121     foreach my $vmid (keys %$list) {
3122         next if $opt_vmid && ($vmid ne $opt_vmid);
3123         next if !$res->{$vmid}->{pid}; # not running
3124         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3125     }
3126
3127     $qmpclient->queue_execute(undef, 2);
3128
3129     foreach my $vmid (keys %$list) {
3130         next if $opt_vmid && ($vmid ne $opt_vmid);
3131         next if !$res->{$vmid}->{pid}; #not running
3132
3133         # we can't use the $qmpclient since it might have already aborted on
3134         # 'query-balloon', but this might also fail for older versions...
3135         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3136         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3137     }
3138
3139     foreach my $vmid (keys %$list) {
3140         next if $opt_vmid && ($vmid ne $opt_vmid);
3141         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3142     }
3143
3144     return $res;
3145 }
3146
3147 sub conf_has_serial {
3148     my ($conf) = @_;
3149
3150     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3151         if ($conf->{"serial$i"}) {
3152             return 1;
3153         }
3154     }
3155
3156     return 0;
3157 }
3158
3159 sub conf_has_audio {
3160     my ($conf, $id) = @_;
3161
3162     $id //= 0;
3163     my $audio = $conf->{"audio$id"};
3164     return if !defined($audio);
3165
3166     my $audioproperties = parse_property_string($audio_fmt, $audio);
3167     my $audiodriver = $audioproperties->{driver} // 'spice';
3168
3169     return {
3170         dev => $audioproperties->{device},
3171         dev_id => "audiodev$id",
3172         backend => $audiodriver,
3173         backend_id => "$audiodriver-backend${id}",
3174     };
3175 }
3176
3177 sub audio_devs {
3178     my ($audio, $audiopciaddr, $machine_version) = @_;
3179
3180     my $devs = [];
3181
3182     my $id = $audio->{dev_id};
3183     my $audiodev = "";
3184     if (min_version($machine_version, 4, 2)) {
3185         $audiodev = ",audiodev=$audio->{backend_id}";
3186     }
3187
3188     if ($audio->{dev} eq 'AC97') {
3189         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3190     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3191         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3192         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3193         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3194     } else {
3195         die "unkown audio device '$audio->{dev}', implement me!";
3196     }
3197
3198     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3199
3200     return $devs;
3201 }
3202
3203 sub get_tpm_paths {
3204     my ($vmid) = @_;
3205     return {
3206         socket => "/var/run/qemu-server/$vmid.swtpm",
3207         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3208     };
3209 }
3210
3211 sub add_tpm_device {
3212     my ($vmid, $devices, $conf) = @_;
3213
3214     return if !$conf->{tpmstate0};
3215
3216     my $paths = get_tpm_paths($vmid);
3217
3218     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3219     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3220     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3221 }
3222
3223 sub start_swtpm {
3224     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3225
3226     return if !$tpmdrive;
3227
3228     my $state;
3229     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3230     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3231     if ($storeid) {
3232         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3233     } else {
3234         $state = $tpm->{file};
3235     }
3236
3237     my $paths = get_tpm_paths($vmid);
3238
3239     # during migration, we will get state from remote
3240     #
3241     if (!$migration) {
3242         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3243         my $setup_cmd = [
3244             "swtpm_setup",
3245             "--tpmstate",
3246             "file://$state",
3247             "--createek",
3248             "--create-ek-cert",
3249             "--create-platform-cert",
3250             "--lock-nvram",
3251             "--config",
3252             "/etc/swtpm_setup.conf", # do not use XDG configs
3253             "--runas",
3254             "0", # force creation as root, error if not possible
3255             "--not-overwrite", # ignore existing state, do not modify
3256         ];
3257
3258         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3259         # TPM 2.0 supports ECC crypto, use if possible
3260         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3261
3262         run_command($setup_cmd, outfunc => sub {
3263             print "swtpm_setup: $1\n";
3264         });
3265     }
3266
3267     my $emulator_cmd = [
3268         "swtpm",
3269         "socket",
3270         "--tpmstate",
3271         "backend-uri=file://$state,mode=0600",
3272         "--ctrl",
3273         "type=unixio,path=$paths->{socket},mode=0600",
3274         "--pid",
3275         "file=$paths->{pid}",
3276         "--terminate", # terminate on QEMU disconnect
3277         "--daemon",
3278     ];
3279     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3280     run_command($emulator_cmd, outfunc => sub { print $1; });
3281
3282     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3283     while (! -e $paths->{pid}) {
3284         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3285         usleep(50_000);
3286     }
3287
3288     # return untainted PID of swtpm daemon so it can be killed on error
3289     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3290     return $1;
3291 }
3292
3293 sub vga_conf_has_spice {
3294     my ($vga) = @_;
3295
3296     my $vgaconf = parse_vga($vga);
3297     my $vgatype = $vgaconf->{type};
3298     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3299
3300     return $1 || 1;
3301 }
3302
3303 sub is_native($) {
3304     my ($arch) = @_;
3305     return get_host_arch() eq $arch;
3306 }
3307
3308 sub get_vm_arch {
3309     my ($conf) = @_;
3310     return $conf->{arch} // get_host_arch();
3311 }
3312
3313 my $default_machines = {
3314     x86_64 => 'pc',
3315     aarch64 => 'virt',
3316 };
3317
3318 sub get_installed_machine_version {
3319     my ($kvmversion) = @_;
3320     $kvmversion = kvm_user_version() if !defined($kvmversion);
3321     $kvmversion =~ m/^(\d+\.\d+)/;
3322     return $1;
3323 }
3324
3325 sub windows_get_pinned_machine_version {
3326     my ($machine, $base_version, $kvmversion) = @_;
3327
3328     my $pin_version = $base_version;
3329     if (!defined($base_version) ||
3330         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3331     ) {
3332         $pin_version = get_installed_machine_version($kvmversion);
3333     }
3334     if (!$machine || $machine eq 'pc') {
3335         $machine = "pc-i440fx-$pin_version";
3336     } elsif ($machine eq 'q35') {
3337         $machine = "pc-q35-$pin_version";
3338     } elsif ($machine eq 'virt') {
3339         $machine = "virt-$pin_version";
3340     } else {
3341         warn "unknown machine type '$machine', not touching that!\n";
3342     }
3343
3344     return $machine;
3345 }
3346
3347 sub get_vm_machine {
3348     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3349
3350     my $machine = $forcemachine || $conf->{machine};
3351
3352     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3353         $kvmversion //= kvm_user_version();
3354         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3355         # layout which confuses windows quite a bit and may result in various regressions..
3356         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3357         if (windows_version($conf->{ostype})) {
3358             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3359         }
3360         $arch //= 'x86_64';
3361         $machine ||= $default_machines->{$arch};
3362         if ($add_pve_version) {
3363             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3364             $machine .= "+pve$pvever";
3365         }
3366     }
3367
3368     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3369         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3370         $machine = $1 if $is_pxe;
3371
3372         # for version-pinned machines that do not include a pve-version (e.g.
3373         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3374         $machine .= '+pve0';
3375
3376         $machine .= '.pxe' if $is_pxe;
3377     }
3378
3379     return $machine;
3380 }
3381
3382 sub get_ovmf_files($$$) {
3383     my ($arch, $efidisk, $smm) = @_;
3384
3385     my $types = $OVMF->{$arch}
3386         or die "no OVMF images known for architecture '$arch'\n";
3387
3388     my $type = 'default';
3389     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3390         $type = $smm ? "4m" : "4m-no-smm";
3391         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3392     }
3393
3394     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3395     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3396     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3397
3398     return ($ovmf_code, $ovmf_vars);
3399 }
3400
3401 my $Arch2Qemu = {
3402     aarch64 => '/usr/bin/qemu-system-aarch64',
3403     x86_64 => '/usr/bin/qemu-system-x86_64',
3404 };
3405 sub get_command_for_arch($) {
3406     my ($arch) = @_;
3407     return '/usr/bin/kvm' if is_native($arch);
3408
3409     my $cmd = $Arch2Qemu->{$arch}
3410         or die "don't know how to emulate architecture '$arch'\n";
3411     return $cmd;
3412 }
3413
3414 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3415 # to use in a QEMU command line (-cpu element), first array_intersect the result
3416 # of query_supported_ with query_understood_. This is necessary because:
3417 #
3418 # a) query_understood_ returns flags the host cannot use and
3419 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3420 #    flags, but CPU settings - with most of them being flags. Those settings
3421 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3422 #
3423 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3424 # expensive. If you need the value returned from this, you can get it much
3425 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3426 # $accel being 'kvm' or 'tcg'.
3427 #
3428 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3429 # changes, automatically populating pmxcfs.
3430 #
3431 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3432 # since kvm and tcg machines support different flags
3433 #
3434 sub query_supported_cpu_flags {
3435     my ($arch) = @_;
3436
3437     $arch //= get_host_arch();
3438     my $default_machine = $default_machines->{$arch};
3439
3440     my $flags = {};
3441
3442     # FIXME: Once this is merged, the code below should work for ARM as well:
3443     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3444     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3445         $arch eq "aarch64";
3446
3447     my $kvm_supported = defined(kvm_version());
3448     my $qemu_cmd = get_command_for_arch($arch);
3449     my $fakevmid = -1;
3450     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3451
3452     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3453     my $query_supported_run_qemu = sub {
3454         my ($kvm) = @_;
3455
3456         my $flags = {};
3457         my $cmd = [
3458             $qemu_cmd,
3459             '-machine', $default_machine,
3460             '-display', 'none',
3461             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3462             '-mon', 'chardev=qmp,mode=control',
3463             '-pidfile', $pidfile,
3464             '-S', '-daemonize'
3465         ];
3466
3467         if (!$kvm) {
3468             push @$cmd, '-accel', 'tcg';
3469         }
3470
3471         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3472         die "QEMU flag querying VM exited with code " . $rc if $rc;
3473
3474         eval {
3475             my $cmd_result = mon_cmd(
3476                 $fakevmid,
3477                 'query-cpu-model-expansion',
3478                 type => 'full',
3479                 model => { name => 'host' }
3480             );
3481
3482             my $props = $cmd_result->{model}->{props};
3483             foreach my $prop (keys %$props) {
3484                 next if $props->{$prop} ne '1';
3485                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3486                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3487                 # We only keep those with underscores, to match /proc/cpuinfo
3488                 $prop =~ s/\.|-/_/g;
3489                 $flags->{$prop} = 1;
3490             }
3491         };
3492         my $err = $@;
3493
3494         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3495         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3496
3497         die $err if $err;
3498
3499         return [ sort keys %$flags ];
3500     };
3501
3502     # We need to query QEMU twice, since KVM and TCG have different supported flags
3503     PVE::QemuConfig->lock_config($fakevmid, sub {
3504         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3505         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3506
3507         if ($kvm_supported) {
3508             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3509             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3510         }
3511     });
3512
3513     return $flags;
3514 }
3515
3516 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3517 my $understood_cpu_flag_dir = "/usr/share/kvm";
3518 sub query_understood_cpu_flags {
3519     my $arch = get_host_arch();
3520     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3521
3522     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3523         if ! -e $filepath;
3524
3525     my $raw = file_get_contents($filepath);
3526     $raw =~ s/^\s+|\s+$//g;
3527     my @flags = split(/\s+/, $raw);
3528
3529     return \@flags;
3530 }
3531
3532 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3533 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3534 my sub should_disable_smm {
3535     my ($conf, $vga) = @_;
3536
3537     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3538         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3539 }
3540
3541 my sub print_ovmf_drive_commandlines {
3542     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3543
3544     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3545
3546     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3547
3548     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3549     if ($d) {
3550         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3551         my ($path, $format) = $d->@{'file', 'format'};
3552         if ($storeid) {
3553             $path = PVE::Storage::path($storecfg, $d->{file});
3554             if (!defined($format)) {
3555                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3556                 $format = qemu_img_format($scfg, $volname);
3557             }
3558         } elsif (!defined($format)) {
3559             die "efidisk format must be specified\n";
3560         }
3561         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3562         if ($path =~ m/^rbd:/) {
3563             $var_drive_str .= ',cache=writeback';
3564             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3565         }
3566         $var_drive_str .= ",format=$format,file=$path";
3567
3568         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3569         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3570     } else {
3571         log_warn("no efidisk configured! Using temporary efivars disk.");
3572         my $path = "/tmp/$vmid-ovmf.fd";
3573         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3574         $var_drive_str .= ",format=raw,file=$path";
3575         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3576     }
3577
3578     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3579 }
3580
3581 sub config_to_command {
3582     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3583         $pbs_backing) = @_;
3584
3585     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3586     my $devices = [];
3587     my $bridges = {};
3588     my $ostype = $conf->{ostype};
3589     my $winversion = windows_version($ostype);
3590     my $kvm = $conf->{kvm};
3591     my $nodename = nodename();
3592
3593     my $arch = get_vm_arch($conf);
3594     my $kvm_binary = get_command_for_arch($arch);
3595     my $kvmver = kvm_user_version($kvm_binary);
3596
3597     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3598         $kvmver //= "undefined";
3599         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3600     }
3601
3602     my $add_pve_version = min_version($kvmver, 4, 1);
3603
3604     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3605     my $machine_version = extract_version($machine_type, $kvmver);
3606     $kvm //= 1 if is_native($arch);
3607
3608     $machine_version =~ m/(\d+)\.(\d+)/;
3609     my ($machine_major, $machine_minor) = ($1, $2);
3610
3611     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3612         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3613     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3614         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3615             ." please upgrade node '$nodename'\n"
3616     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3617         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3618         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3619             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3620             ." node '$nodename'\n";
3621     }
3622
3623     # if a specific +pve version is required for a feature, use $version_guard
3624     # instead of min_version to allow machines to be run with the minimum
3625     # required version
3626     my $required_pve_version = 0;
3627     my $version_guard = sub {
3628         my ($major, $minor, $pve) = @_;
3629         return 0 if !min_version($machine_version, $major, $minor, $pve);
3630         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3631         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3632         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3633         return 1;
3634     };
3635
3636     if ($kvm && !defined kvm_version()) {
3637         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3638             ." or enable in BIOS.\n";
3639     }
3640
3641     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3642     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3643     my $use_old_bios_files = undef;
3644     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3645
3646     my $cmd = [];
3647     if ($conf->{affinity}) {
3648         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3649     }
3650
3651     push @$cmd, $kvm_binary;
3652
3653     push @$cmd, '-id', $vmid;
3654
3655     my $vmname = $conf->{name} || "vm$vmid";
3656
3657     push @$cmd, '-name', "$vmname,debug-threads=on";
3658
3659     push @$cmd, '-no-shutdown';
3660
3661     my $use_virtio = 0;
3662
3663     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3664     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3665     push @$cmd, '-mon', "chardev=qmp,mode=control";
3666
3667     if (min_version($machine_version, 2, 12)) {
3668         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3669         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3670     }
3671
3672     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3673
3674     push @$cmd, '-daemonize';
3675
3676     if ($conf->{smbios1}) {
3677         my $smbios_conf = parse_smbios1($conf->{smbios1});
3678         if ($smbios_conf->{base64}) {
3679             # Do not pass base64 flag to qemu
3680             delete $smbios_conf->{base64};
3681             my $smbios_string = "";
3682             foreach my $key (keys %$smbios_conf) {
3683                 my $value;
3684                 if ($key eq "uuid") {
3685                     $value = $smbios_conf->{uuid}
3686                 } else {
3687                     $value = decode_base64($smbios_conf->{$key});
3688                 }
3689                 # qemu accepts any binary data, only commas need escaping by double comma
3690                 $value =~ s/,/,,/g;
3691                 $smbios_string .= "," . $key . "=" . $value if $value;
3692             }
3693             push @$cmd, '-smbios', "type=1" . $smbios_string;
3694         } else {
3695             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3696         }
3697     }
3698
3699     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3700         my ($code_drive_str, $var_drive_str) =
3701             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3702         push $cmd->@*, '-drive', $code_drive_str;
3703         push $cmd->@*, '-drive', $var_drive_str;
3704     }
3705
3706     if ($q35) { # tell QEMU to load q35 config early
3707         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3708         if (min_version($machine_version, 4, 0)) {
3709             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3710         } else {
3711             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3712         }
3713     }
3714
3715     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3716         push @$cmd, $fixups->@*;
3717     }
3718
3719     if ($conf->{vmgenid}) {
3720         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3721     }
3722
3723     # add usb controllers
3724     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3725         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3726     push @$devices, @usbcontrollers if @usbcontrollers;
3727     my $vga = parse_vga($conf->{vga});
3728
3729     my $qxlnum = vga_conf_has_spice($conf->{vga});
3730     $vga->{type} = 'qxl' if $qxlnum;
3731
3732     if (!$vga->{type}) {
3733         if ($arch eq 'aarch64') {
3734             $vga->{type} = 'virtio';
3735         } elsif (min_version($machine_version, 2, 9)) {
3736             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3737         } else {
3738             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3739         }
3740     }
3741
3742     # enable absolute mouse coordinates (needed by vnc)
3743     my $tablet = $conf->{tablet};
3744     if (!defined($tablet)) {
3745         $tablet = $defaults->{tablet};
3746         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3747         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3748     }
3749
3750     if ($tablet) {
3751         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3752         my $kbd = print_keyboarddevice_full($conf, $arch);
3753         push @$devices, '-device', $kbd if defined($kbd);
3754     }
3755
3756     my $bootorder = device_bootorder($conf);
3757
3758     # host pci device passthrough
3759     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3760         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3761
3762     # usb devices
3763     my $usb_dev_features = {};
3764     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3765
3766     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3767         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3768     push @$devices, @usbdevices if @usbdevices;
3769
3770     # serial devices
3771     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3772         my $path = $conf->{"serial$i"} or next;
3773         if ($path eq 'socket') {
3774             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3775             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3776             # On aarch64, serial0 is the UART device. QEMU only allows
3777             # connecting UART devices via the '-serial' command line, as
3778             # the device has a fixed slot on the hardware...
3779             if ($arch eq 'aarch64' && $i == 0) {
3780                 push @$devices, '-serial', "chardev:serial$i";
3781             } else {
3782                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3783             }
3784         } else {
3785             die "no such serial device\n" if ! -c $path;
3786             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3787             push @$devices, '-device', "isa-serial,chardev=serial$i";
3788         }
3789     }
3790
3791     # parallel devices
3792     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3793         if (my $path = $conf->{"parallel$i"}) {
3794             die "no such parallel device\n" if ! -c $path;
3795             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3796             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3797             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3798         }
3799     }
3800
3801     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3802         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3803         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3804         push @$devices, @$audio_devs;
3805     }
3806
3807     add_tpm_device($vmid, $devices, $conf);
3808
3809     my $sockets = 1;
3810     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3811     $sockets = $conf->{sockets} if  $conf->{sockets};
3812
3813     my $cores = $conf->{cores} || 1;
3814
3815     my $maxcpus = $sockets * $cores;
3816
3817     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3818
3819     my $allowed_vcpus = $cpuinfo->{cpus};
3820
3821     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3822
3823     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3824         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3825         for (my $i = 2; $i <= $vcpus; $i++)  {
3826             my $cpustr = print_cpu_device($conf,$i);
3827             push @$cmd, '-device', $cpustr;
3828         }
3829
3830     } else {
3831
3832         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3833     }
3834     push @$cmd, '-nodefaults';
3835
3836     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3837
3838     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3839
3840     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3841
3842     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3843         push @$devices, '-device', print_vga_device(
3844             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3845
3846         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3847
3848         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3849         push @$cmd,  '-vnc', "unix:$socket,password=on";
3850     } else {
3851         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3852         push @$cmd, '-nographic';
3853     }
3854
3855     # time drift fix
3856     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3857     my $useLocaltime = $conf->{localtime};
3858
3859     if ($winversion >= 5) { # windows
3860         $useLocaltime = 1 if !defined($conf->{localtime});
3861
3862         # use time drift fix when acpi is enabled
3863         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3864             $tdf = 1 if !defined($conf->{tdf});
3865         }
3866     }
3867
3868     if ($winversion >= 6) {
3869         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3870         push @$cmd, '-no-hpet';
3871     }
3872
3873     push @$rtcFlags, 'driftfix=slew' if $tdf;
3874
3875     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3876         push @$rtcFlags, "base=$conf->{startdate}";
3877     } elsif ($useLocaltime) {
3878         push @$rtcFlags, 'base=localtime';
3879     }
3880
3881     if ($forcecpu) {
3882         push @$cmd, '-cpu', $forcecpu;
3883     } else {
3884         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3885     }
3886
3887     PVE::QemuServer::Memory::config(
3888         $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory}, $cmd);
3889
3890     push @$cmd, '-S' if $conf->{freeze};
3891
3892     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3893
3894     my $guest_agent = parse_guest_agent($conf);
3895
3896     if ($guest_agent->{enabled}) {
3897         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3898         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3899
3900         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3901             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3902             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3903             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3904         } elsif ($guest_agent->{type} eq 'isa') {
3905             push @$devices, '-device', "isa-serial,chardev=qga0";
3906         }
3907     }
3908
3909     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3910     if ($rng && $version_guard->(4, 1, 2)) {
3911         check_rng_source($rng->{source});
3912
3913         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3914         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3915         my $limiter_str = "";
3916         if ($max_bytes) {
3917             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3918         }
3919
3920         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3921         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3922         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3923     }
3924
3925     my $spice_port;
3926
3927     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3928         if ($qxlnum > 1) {
3929             if ($winversion){
3930                 for (my $i = 1; $i < $qxlnum; $i++){
3931                     push @$devices, '-device', print_vga_device(
3932                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3933                 }
3934             } else {
3935                 # assume other OS works like Linux
3936                 my ($ram, $vram) = ("134217728", "67108864");
3937                 if ($vga->{memory}) {
3938                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3939                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3940                 }
3941                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3942                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3943             }
3944         }
3945
3946         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3947
3948         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3949         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3950         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3951
3952         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3953         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3954         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3955
3956         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3957         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3958
3959         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3960         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3961         if ($spice_enhancement->{foldersharing}) {
3962             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3963             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3964         }
3965
3966         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3967         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3968             if $spice_enhancement->{videostreaming};
3969
3970         push @$devices, '-spice', "$spice_opts";
3971     }
3972
3973     # enable balloon by default, unless explicitly disabled
3974     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3975         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3976         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3977         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3978         push @$devices, '-device', $ballooncmd;
3979     }
3980
3981     if ($conf->{watchdog}) {
3982         my $wdopts = parse_watchdog($conf->{watchdog});
3983         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3984         my $watchdog = $wdopts->{model} || 'i6300esb';
3985         push @$devices, '-device', "$watchdog$pciaddr";
3986         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3987     }
3988
3989     my $vollist = [];
3990     my $scsicontroller = {};
3991     my $ahcicontroller = {};
3992     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3993
3994     # Add iscsi initiator name if available
3995     if (my $initiator = get_initiator_name()) {
3996         push @$devices, '-iscsi', "initiator-name=$initiator";
3997     }
3998
3999     PVE::QemuConfig->foreach_volume($conf, sub {
4000         my ($ds, $drive) = @_;
4001
4002         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4003             check_volume_storage_type($storecfg, $drive->{file});
4004             push @$vollist, $drive->{file};
4005         }
4006
4007         # ignore efidisk here, already added in bios/fw handling code above
4008         return if $drive->{interface} eq 'efidisk';
4009         # similar for TPM
4010         return if $drive->{interface} eq 'tpmstate';
4011
4012         $use_virtio = 1 if $ds =~ m/^virtio/;
4013
4014         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4015
4016         if ($drive->{interface} eq 'virtio'){
4017            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4018         }
4019
4020         if ($drive->{interface} eq 'scsi') {
4021
4022             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4023
4024             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4025                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4026
4027             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4028             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4029
4030             my $iothread = '';
4031             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4032                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4033                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4034             } elsif ($drive->{iothread}) {
4035                 log_warn(
4036                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4037                 );
4038             }
4039
4040             my $queues = '';
4041             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4042                 $queues = ",num_queues=$drive->{queues}";
4043             }
4044
4045             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4046                 if !$scsicontroller->{$controller};
4047             $scsicontroller->{$controller}=1;
4048         }
4049
4050         if ($drive->{interface} eq 'sata') {
4051             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4052             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4053             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4054                 if !$ahcicontroller->{$controller};
4055             $ahcicontroller->{$controller}=1;
4056         }
4057
4058         my $pbs_conf = $pbs_backing->{$ds};
4059         my $pbs_name = undef;
4060         if ($pbs_conf) {
4061             $pbs_name = "drive-$ds-pbs";
4062             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4063         }
4064
4065         my $drive_cmd = print_drive_commandline_full(
4066             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4067
4068         # extra protection for templates, but SATA and IDE don't support it..
4069         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4070
4071         push @$devices, '-drive',$drive_cmd;
4072         push @$devices, '-device', print_drivedevice_full(
4073             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4074     });
4075
4076     for (my $i = 0; $i < $MAX_NETS; $i++) {
4077         my $netname = "net$i";
4078
4079         next if !$conf->{$netname};
4080         my $d = parse_net($conf->{$netname});
4081         next if !$d;
4082         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4083
4084         $use_virtio = 1 if $d->{model} eq 'virtio';
4085
4086         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4087
4088         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4089         push @$devices, '-netdev', $netdevfull;
4090
4091         my $netdevicefull = print_netdevice_full(
4092             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4093
4094         push @$devices, '-device', $netdevicefull;
4095     }
4096
4097     if ($conf->{ivshmem}) {
4098         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4099
4100         my $bus;
4101         if ($q35) {
4102             $bus = print_pcie_addr("ivshmem");
4103         } else {
4104             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4105         }
4106
4107         my $ivshmem_name = $ivshmem->{name} // $vmid;
4108         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4109
4110         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4111         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4112             .",size=$ivshmem->{size}M";
4113     }
4114
4115     # pci.4 is nested in pci.1
4116     $bridges->{1} = 1 if $bridges->{4};
4117
4118     if (!$q35) { # add pci bridges
4119         if (min_version($machine_version, 2, 3)) {
4120            $bridges->{1} = 1;
4121            $bridges->{2} = 1;
4122         }
4123         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4124     }
4125
4126     for my $k (sort {$b cmp $a} keys %$bridges) {
4127         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4128
4129         my $k_name = $k;
4130         if ($k == 2 && $legacy_igd) {
4131             $k_name = "$k-igd";
4132         }
4133         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4134         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4135
4136         if ($q35) { # add after -readconfig pve-q35.cfg
4137             splice @$devices, 2, 0, '-device', $devstr;
4138         } else {
4139             unshift @$devices, '-device', $devstr if $k > 0;
4140         }
4141     }
4142
4143     if (!$kvm) {
4144         push @$machineFlags, 'accel=tcg';
4145     }
4146
4147     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4148
4149     my $machine_type_min = $machine_type;
4150     if ($add_pve_version) {
4151         $machine_type_min =~ s/\+pve\d+$//;
4152         $machine_type_min .= "+pve$required_pve_version";
4153     }
4154     push @$machineFlags, "type=${machine_type_min}";
4155
4156     push @$cmd, @$devices;
4157     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4158     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4159     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4160
4161     if (my $vmstate = $conf->{vmstate}) {
4162         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4163         push @$vollist, $vmstate;
4164         push @$cmd, '-loadstate', $statepath;
4165         print "activating and using '$vmstate' as vmstate\n";
4166     }
4167
4168     if (PVE::QemuConfig->is_template($conf)) {
4169         # needed to workaround base volumes being read-only
4170         push @$cmd, '-snapshot';
4171     }
4172
4173     # add custom args
4174     if ($conf->{args}) {
4175         my $aa = PVE::Tools::split_args($conf->{args});
4176         push @$cmd, @$aa;
4177     }
4178
4179     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4180 }
4181
4182 sub check_rng_source {
4183     my ($source) = @_;
4184
4185     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4186     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4187         if ! -e $source;
4188
4189     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4190     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4191         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4192         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4193         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4194             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4195             ." to the host.\n";
4196     }
4197 }
4198
4199 sub spice_port {
4200     my ($vmid) = @_;
4201
4202     my $res = mon_cmd($vmid, 'query-spice');
4203
4204     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4205 }
4206
4207 sub vm_devices_list {
4208     my ($vmid) = @_;
4209
4210     my $res = mon_cmd($vmid, 'query-pci');
4211     my $devices_to_check = [];
4212     my $devices = {};
4213     foreach my $pcibus (@$res) {
4214         push @$devices_to_check, @{$pcibus->{devices}},
4215     }
4216
4217     while (@$devices_to_check) {
4218         my $to_check = [];
4219         for my $d (@$devices_to_check) {
4220             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4221             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4222
4223             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4224             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4225         }
4226         $devices_to_check = $to_check;
4227     }
4228
4229     my $resblock = mon_cmd($vmid, 'query-block');
4230     foreach my $block (@$resblock) {
4231         if($block->{device} =~ m/^drive-(\S+)/){
4232                 $devices->{$1} = 1;
4233         }
4234     }
4235
4236     my $resmice = mon_cmd($vmid, 'query-mice');
4237     foreach my $mice (@$resmice) {
4238         if ($mice->{name} eq 'QEMU HID Tablet') {
4239             $devices->{tablet} = 1;
4240             last;
4241         }
4242     }
4243
4244     # for usb devices there is no query-usb
4245     # but we can iterate over the entries in
4246     # qom-list path=/machine/peripheral
4247     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4248     foreach my $per (@$resperipheral) {
4249         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4250             $devices->{$per->{name}} = 1;
4251         }
4252     }
4253
4254     return $devices;
4255 }
4256
4257 sub vm_deviceplug {
4258     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4259
4260     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4261
4262     my $devices_list = vm_devices_list($vmid);
4263     return 1 if defined($devices_list->{$deviceid});
4264
4265     # add PCI bridge if we need it for the device
4266     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4267
4268     if ($deviceid eq 'tablet') {
4269         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4270     } elsif ($deviceid eq 'keyboard') {
4271         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4272     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4273         my $id = $1;
4274         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4275         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4276     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4277         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4278     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4279         qemu_iothread_add($vmid, $deviceid, $device);
4280
4281         qemu_driveadd($storecfg, $vmid, $device);
4282         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4283
4284         qemu_deviceadd($vmid, $devicefull);
4285         eval { qemu_deviceaddverify($vmid, $deviceid); };
4286         if (my $err = $@) {
4287             eval { qemu_drivedel($vmid, $deviceid); };
4288             warn $@ if $@;
4289             die $err;
4290         }
4291     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4292         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4293         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4294         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4295
4296         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4297
4298         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4299             qemu_iothread_add($vmid, $deviceid, $device);
4300             $devicefull .= ",iothread=iothread-$deviceid";
4301         }
4302
4303         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4304             $devicefull .= ",num_queues=$device->{queues}";
4305         }
4306
4307         qemu_deviceadd($vmid, $devicefull);
4308         qemu_deviceaddverify($vmid, $deviceid);
4309     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4310         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4311         qemu_driveadd($storecfg, $vmid, $device);
4312
4313         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4314         eval { qemu_deviceadd($vmid, $devicefull); };
4315         if (my $err = $@) {
4316             eval { qemu_drivedel($vmid, $deviceid); };
4317             warn $@ if $@;
4318             die $err;
4319         }
4320     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4321         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4322
4323         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4324         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4325         my $use_old_bios_files = undef;
4326         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4327
4328         my $netdevicefull = print_netdevice_full(
4329             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4330         qemu_deviceadd($vmid, $netdevicefull);
4331         eval {
4332             qemu_deviceaddverify($vmid, $deviceid);
4333             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4334         };
4335         if (my $err = $@) {
4336             eval { qemu_netdevdel($vmid, $deviceid); };
4337             warn $@ if $@;
4338             die $err;
4339         }
4340     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4341         my $bridgeid = $2;
4342         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4343         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4344
4345         qemu_deviceadd($vmid, $devicefull);
4346         qemu_deviceaddverify($vmid, $deviceid);
4347     } else {
4348         die "can't hotplug device '$deviceid'\n";
4349     }
4350
4351     return 1;
4352 }
4353
4354 # fixme: this should raise exceptions on error!
4355 sub vm_deviceunplug {
4356     my ($vmid, $conf, $deviceid) = @_;
4357
4358     my $devices_list = vm_devices_list($vmid);
4359     return 1 if !defined($devices_list->{$deviceid});
4360
4361     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4362     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4363
4364     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4365         qemu_devicedel($vmid, $deviceid);
4366     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4367         qemu_devicedel($vmid, $deviceid);
4368         qemu_devicedelverify($vmid, $deviceid);
4369     } elsif ($deviceid =~ m/^usb\d+$/) {
4370         qemu_devicedel($vmid, $deviceid);
4371         qemu_devicedelverify($vmid, $deviceid);
4372     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4373         my $device = parse_drive($deviceid, $conf->{$deviceid});
4374
4375         qemu_devicedel($vmid, $deviceid);
4376         qemu_devicedelverify($vmid, $deviceid);
4377         qemu_drivedel($vmid, $deviceid);
4378         qemu_iothread_del($vmid, $deviceid, $device);
4379     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4380         qemu_devicedel($vmid, $deviceid);
4381         qemu_devicedelverify($vmid, $deviceid);
4382     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4383         my $device = parse_drive($deviceid, $conf->{$deviceid});
4384
4385         qemu_devicedel($vmid, $deviceid);
4386         qemu_devicedelverify($vmid, $deviceid);
4387         qemu_drivedel($vmid, $deviceid);
4388         qemu_deletescsihw($conf, $vmid, $deviceid);
4389
4390         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4391             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4392     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4393         qemu_devicedel($vmid, $deviceid);
4394         qemu_devicedelverify($vmid, $deviceid);
4395         qemu_netdevdel($vmid, $deviceid);
4396     } else {
4397         die "can't unplug device '$deviceid'\n";
4398     }
4399
4400     return 1;
4401 }
4402
4403 sub qemu_spice_usbredir_chardev_add {
4404     my ($vmid, $id) = @_;
4405
4406     mon_cmd($vmid, "chardev-add" , (
4407         id => $id,
4408         backend => {
4409             type => 'spicevmc',
4410             data => {
4411                 type => "usbredir",
4412             },
4413         },
4414     ));
4415 }
4416
4417 sub qemu_deviceadd {
4418     my ($vmid, $devicefull) = @_;
4419
4420     $devicefull = "driver=".$devicefull;
4421     my %options =  split(/[=,]/, $devicefull);
4422
4423     mon_cmd($vmid, "device_add" , %options);
4424 }
4425
4426 sub qemu_devicedel {
4427     my ($vmid, $deviceid) = @_;
4428
4429     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4430 }
4431
4432 sub qemu_iothread_add {
4433     my ($vmid, $deviceid, $device) = @_;
4434
4435     if ($device->{iothread}) {
4436         my $iothreads = vm_iothreads_list($vmid);
4437         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4438     }
4439 }
4440
4441 sub qemu_iothread_del {
4442     my ($vmid, $deviceid, $device) = @_;
4443
4444     if ($device->{iothread}) {
4445         my $iothreads = vm_iothreads_list($vmid);
4446         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4447     }
4448 }
4449
4450 sub qemu_objectadd {
4451     my ($vmid, $objectid, $qomtype) = @_;
4452
4453     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4454
4455     return 1;
4456 }
4457
4458 sub qemu_objectdel {
4459     my ($vmid, $objectid) = @_;
4460
4461     mon_cmd($vmid, "object-del", id => $objectid);
4462
4463     return 1;
4464 }
4465
4466 sub qemu_driveadd {
4467     my ($storecfg, $vmid, $device) = @_;
4468
4469     my $kvmver = get_running_qemu_version($vmid);
4470     my $io_uring = min_version($kvmver, 6, 0);
4471     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4472     $drive =~ s/\\/\\\\/g;
4473     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4474
4475     # If the command succeeds qemu prints: "OK"
4476     return 1 if $ret =~ m/OK/s;
4477
4478     die "adding drive failed: $ret\n";
4479 }
4480
4481 sub qemu_drivedel {
4482     my ($vmid, $deviceid) = @_;
4483
4484     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4485     $ret =~ s/^\s+//;
4486
4487     return 1 if $ret eq "";
4488
4489     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4490     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4491
4492     die "deleting drive $deviceid failed : $ret\n";
4493 }
4494
4495 sub qemu_deviceaddverify {
4496     my ($vmid, $deviceid) = @_;
4497
4498     for (my $i = 0; $i <= 5; $i++) {
4499          my $devices_list = vm_devices_list($vmid);
4500          return 1 if defined($devices_list->{$deviceid});
4501          sleep 1;
4502     }
4503
4504     die "error on hotplug device '$deviceid'\n";
4505 }
4506
4507
4508 sub qemu_devicedelverify {
4509     my ($vmid, $deviceid) = @_;
4510
4511     # need to verify that the device is correctly removed as device_del
4512     # is async and empty return is not reliable
4513
4514     for (my $i = 0; $i <= 5; $i++) {
4515          my $devices_list = vm_devices_list($vmid);
4516          return 1 if !defined($devices_list->{$deviceid});
4517          sleep 1;
4518     }
4519
4520     die "error on hot-unplugging device '$deviceid'\n";
4521 }
4522
4523 sub qemu_findorcreatescsihw {
4524     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4525
4526     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4527
4528     my $scsihwid="$controller_prefix$controller";
4529     my $devices_list = vm_devices_list($vmid);
4530
4531     if (!defined($devices_list->{$scsihwid})) {
4532         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4533     }
4534
4535     return 1;
4536 }
4537
4538 sub qemu_deletescsihw {
4539     my ($conf, $vmid, $opt) = @_;
4540
4541     my $device = parse_drive($opt, $conf->{$opt});
4542
4543     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4544         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4545         return 1;
4546     }
4547
4548     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4549
4550     my $devices_list = vm_devices_list($vmid);
4551     foreach my $opt (keys %{$devices_list}) {
4552         if (is_valid_drivename($opt)) {
4553             my $drive = parse_drive($opt, $conf->{$opt});
4554             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4555                 return 1;
4556             }
4557         }
4558     }
4559
4560     my $scsihwid="scsihw$controller";
4561
4562     vm_deviceunplug($vmid, $conf, $scsihwid);
4563
4564     return 1;
4565 }
4566
4567 sub qemu_add_pci_bridge {
4568     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4569
4570     my $bridges = {};
4571
4572     my $bridgeid;
4573
4574     print_pci_addr($device, $bridges, $arch, $machine_type);
4575
4576     while (my ($k, $v) = each %$bridges) {
4577         $bridgeid = $k;
4578     }
4579     return 1 if !defined($bridgeid) || $bridgeid < 1;
4580
4581     my $bridge = "pci.$bridgeid";
4582     my $devices_list = vm_devices_list($vmid);
4583
4584     if (!defined($devices_list->{$bridge})) {
4585         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4586     }
4587
4588     return 1;
4589 }
4590
4591 sub qemu_set_link_status {
4592     my ($vmid, $device, $up) = @_;
4593
4594     mon_cmd($vmid, "set_link", name => $device,
4595                up => $up ? JSON::true : JSON::false);
4596 }
4597
4598 sub qemu_netdevadd {
4599     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4600
4601     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4602     my %options =  split(/[=,]/, $netdev);
4603
4604     if (defined(my $vhost = $options{vhost})) {
4605         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4606     }
4607
4608     if (defined(my $queues = $options{queues})) {
4609         $options{queues} = $queues + 0;
4610     }
4611
4612     mon_cmd($vmid, "netdev_add",  %options);
4613     return 1;
4614 }
4615
4616 sub qemu_netdevdel {
4617     my ($vmid, $deviceid) = @_;
4618
4619     mon_cmd($vmid, "netdev_del", id => $deviceid);
4620 }
4621
4622 sub qemu_usb_hotplug {
4623     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4624
4625     return if !$device;
4626
4627     # remove the old one first
4628     vm_deviceunplug($vmid, $conf, $deviceid);
4629
4630     # check if xhci controller is necessary and available
4631     my $devicelist = vm_devices_list($vmid);
4632
4633     if (!$devicelist->{xhci}) {
4634         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4635         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4636     }
4637
4638     # print_usbdevice_full expects the parsed device
4639     my $d = parse_usb_device($device->{host});
4640     $d->{usb3} = $device->{usb3};
4641
4642     # add the new one
4643     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4644 }
4645
4646 sub qemu_cpu_hotplug {
4647     my ($vmid, $conf, $vcpus) = @_;
4648
4649     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4650
4651     my $sockets = 1;
4652     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4653     $sockets = $conf->{sockets} if  $conf->{sockets};
4654     my $cores = $conf->{cores} || 1;
4655     my $maxcpus = $sockets * $cores;
4656
4657     $vcpus = $maxcpus if !$vcpus;
4658
4659     die "you can't add more vcpus than maxcpus\n"
4660         if $vcpus > $maxcpus;
4661
4662     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4663
4664     if ($vcpus < $currentvcpus) {
4665
4666         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4667
4668             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4669                 qemu_devicedel($vmid, "cpu$i");
4670                 my $retry = 0;
4671                 my $currentrunningvcpus = undef;
4672                 while (1) {
4673                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4674                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4675                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4676                     $retry++;
4677                     sleep 1;
4678                 }
4679                 #update conf after each succesfull cpu unplug
4680                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4681                 PVE::QemuConfig->write_config($vmid, $conf);
4682             }
4683         } else {
4684             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4685         }
4686
4687         return;
4688     }
4689
4690     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4691     die "vcpus in running vm does not match its configuration\n"
4692         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4693
4694     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4695
4696         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4697             my $cpustr = print_cpu_device($conf, $i);
4698             qemu_deviceadd($vmid, $cpustr);
4699
4700             my $retry = 0;
4701             my $currentrunningvcpus = undef;
4702             while (1) {
4703                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4704                 last if scalar(@{$currentrunningvcpus}) == $i;
4705                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4706                 sleep 1;
4707                 $retry++;
4708             }
4709             #update conf after each succesfull cpu hotplug
4710             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4711             PVE::QemuConfig->write_config($vmid, $conf);
4712         }
4713     } else {
4714
4715         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4716             mon_cmd($vmid, "cpu-add", id => int($i));
4717         }
4718     }
4719 }
4720
4721 sub qemu_block_set_io_throttle {
4722     my ($vmid, $deviceid,
4723         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4724         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4725         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4726         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4727
4728     return if !check_running($vmid) ;
4729
4730     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4731         bps => int($bps),
4732         bps_rd => int($bps_rd),
4733         bps_wr => int($bps_wr),
4734         iops => int($iops),
4735         iops_rd => int($iops_rd),
4736         iops_wr => int($iops_wr),
4737         bps_max => int($bps_max),
4738         bps_rd_max => int($bps_rd_max),
4739         bps_wr_max => int($bps_wr_max),
4740         iops_max => int($iops_max),
4741         iops_rd_max => int($iops_rd_max),
4742         iops_wr_max => int($iops_wr_max),
4743         bps_max_length => int($bps_max_length),
4744         bps_rd_max_length => int($bps_rd_max_length),
4745         bps_wr_max_length => int($bps_wr_max_length),
4746         iops_max_length => int($iops_max_length),
4747         iops_rd_max_length => int($iops_rd_max_length),
4748         iops_wr_max_length => int($iops_wr_max_length),
4749     );
4750
4751 }
4752
4753 sub qemu_block_resize {
4754     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4755
4756     my $running = check_running($vmid);
4757
4758     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4759
4760     return if !$running;
4761
4762     my $padding = (1024 - $size % 1024) % 1024;
4763     $size = $size + $padding;
4764
4765     mon_cmd(
4766         $vmid,
4767         "block_resize",
4768         device => $deviceid,
4769         size => int($size),
4770         timeout => 60,
4771     );
4772 }
4773
4774 sub qemu_volume_snapshot {
4775     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4776
4777     my $running = check_running($vmid);
4778
4779     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4780         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4781     } else {
4782         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4783     }
4784 }
4785
4786 sub qemu_volume_snapshot_delete {
4787     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4788
4789     my $running = check_running($vmid);
4790
4791     if($running) {
4792
4793         $running = undef;
4794         my $conf = PVE::QemuConfig->load_config($vmid);
4795         PVE::QemuConfig->foreach_volume($conf, sub {
4796             my ($ds, $drive) = @_;
4797             $running = 1 if $drive->{file} eq $volid;
4798         });
4799     }
4800
4801     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4802         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4803     } else {
4804         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4805     }
4806 }
4807
4808 sub set_migration_caps {
4809     my ($vmid, $savevm) = @_;
4810
4811     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4812
4813     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4814     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4815
4816     my $cap_ref = [];
4817
4818     my $enabled_cap = {
4819         "auto-converge" => 1,
4820         "xbzrle" => 1,
4821         "x-rdma-pin-all" => 0,
4822         "zero-blocks" => 0,
4823         "compress" => 0,
4824         "dirty-bitmaps" => $dirty_bitmaps,
4825     };
4826
4827     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4828
4829     for my $supported_capability (@$supported_capabilities) {
4830         push @$cap_ref, {
4831             capability => $supported_capability->{capability},
4832             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4833         };
4834     }
4835
4836     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4837 }
4838
4839 sub foreach_volid {
4840     my ($conf, $func, @param) = @_;
4841
4842     my $volhash = {};
4843
4844     my $test_volid = sub {
4845         my ($key, $drive, $snapname) = @_;
4846
4847         my $volid = $drive->{file};
4848         return if !$volid;
4849
4850         $volhash->{$volid}->{cdrom} //= 1;
4851         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4852
4853         my $replicate = $drive->{replicate} // 1;
4854         $volhash->{$volid}->{replicate} //= 0;
4855         $volhash->{$volid}->{replicate} = 1 if $replicate;
4856
4857         $volhash->{$volid}->{shared} //= 0;
4858         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4859
4860         $volhash->{$volid}->{referenced_in_config} //= 0;
4861         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4862
4863         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4864             if defined($snapname);
4865
4866         my $size = $drive->{size};
4867         $volhash->{$volid}->{size} //= $size if $size;
4868
4869         $volhash->{$volid}->{is_vmstate} //= 0;
4870         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4871
4872         $volhash->{$volid}->{is_tpmstate} //= 0;
4873         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4874
4875         $volhash->{$volid}->{is_unused} //= 0;
4876         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4877
4878         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4879     };
4880
4881     my $include_opts = {
4882         extra_keys => ['vmstate'],
4883         include_unused => 1,
4884     };
4885
4886     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4887     foreach my $snapname (keys %{$conf->{snapshots}}) {
4888         my $snap = $conf->{snapshots}->{$snapname};
4889         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4890     }
4891
4892     foreach my $volid (keys %$volhash) {
4893         &$func($volid, $volhash->{$volid}, @param);
4894     }
4895 }
4896
4897 my $fast_plug_option = {
4898     'lock' => 1,
4899     'name' => 1,
4900     'onboot' => 1,
4901     'shares' => 1,
4902     'startup' => 1,
4903     'description' => 1,
4904     'protection' => 1,
4905     'vmstatestorage' => 1,
4906     'hookscript' => 1,
4907     'tags' => 1,
4908 };
4909
4910 for my $opt (keys %$confdesc_cloudinit) {
4911     $fast_plug_option->{$opt} = 1;
4912 };
4913
4914 # hotplug changes in [PENDING]
4915 # $selection hash can be used to only apply specified options, for
4916 # example: { cores => 1 } (only apply changed 'cores')
4917 # $errors ref is used to return error messages
4918 sub vmconfig_hotplug_pending {
4919     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4920
4921     my $defaults = load_defaults();
4922     my $arch = get_vm_arch($conf);
4923     my $machine_type = get_vm_machine($conf, undef, $arch);
4924
4925     # commit values which do not have any impact on running VM first
4926     # Note: those option cannot raise errors, we we do not care about
4927     # $selection and always apply them.
4928
4929     my $add_error = sub {
4930         my ($opt, $msg) = @_;
4931         $errors->{$opt} = "hotplug problem - $msg";
4932     };
4933
4934     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4935
4936     my $cloudinit_record_changed = sub {
4937         my ($conf, $opt, $old, $new) = @_;
4938         return if !$cloudinit_pending_properties->{$opt};
4939
4940         my $ci = ($conf->{cloudinit} //= {});
4941
4942         my $recorded = $ci->{$opt};
4943         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4944
4945         if (defined($new)) {
4946             if (defined($old)) {
4947                 # an existing value is being modified
4948                 if (defined($recorded)) {
4949                     # the value was already not in sync
4950                     if ($new eq $recorded) {
4951                         # a value is being reverted to the cloud-init state:
4952                         delete $ci->{$opt};
4953                         delete $added{$opt};
4954                     } else {
4955                         # the value was changed multiple times, do nothing
4956                     }
4957                 } elsif ($added{$opt}) {
4958                     # the value had been marked as added and is being changed, do nothing
4959                 } else {
4960                     # the value is new, record it:
4961                     $ci->{$opt} = $old;
4962                 }
4963             } else {
4964                 # a new value is being added
4965                 if (defined($recorded)) {
4966                     # it was already not in sync
4967                     if ($new eq $recorded) {
4968                         # a value is being reverted to the cloud-init state:
4969                         delete $ci->{$opt};
4970                         delete $added{$opt};
4971                     } else {
4972                         # the value had temporarily been removed, do nothing
4973                     }
4974                 } elsif ($added{$opt}) {
4975                     # the value had been marked as added already, do nothing
4976                 } else {
4977                     # the value is new, add it
4978                     $added{$opt} = 1;
4979                 }
4980             }
4981         } elsif (!defined($old)) {
4982             # a non-existent value is being removed? ignore...
4983         } else {
4984             # a value is being deleted
4985             if (defined($recorded)) {
4986                 # a value was already recorded, just keep it
4987             } elsif ($added{$opt}) {
4988                 # the value was marked as added, remove it
4989                 delete $added{$opt};
4990             } else {
4991                 # a previously unrecorded value is being removed, record the old value:
4992                 $ci->{$opt} = $old;
4993             }
4994         }
4995
4996         my $added = join(',', sort keys %added);
4997         $ci->{added} = $added if length($added);
4998     };
4999
5000     my $changes = 0;
5001     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5002         if ($fast_plug_option->{$opt}) {
5003             my $new = delete $conf->{pending}->{$opt};
5004             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
5005             $conf->{$opt} = $new;
5006             $changes = 1;
5007         }
5008     }
5009
5010     if ($changes) {
5011         PVE::QemuConfig->write_config($vmid, $conf);
5012     }
5013
5014     my $ostype = $conf->{ostype};
5015     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5016     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5017     my $usb_hotplug = $hotplug_features->{usb}
5018         && min_version($version, 7, 1)
5019         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5020
5021     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5022     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5023
5024     foreach my $opt (sort keys %$pending_delete_hash) {
5025         next if $selection && !$selection->{$opt};
5026         my $force = $pending_delete_hash->{$opt}->{force};
5027         eval {
5028             if ($opt eq 'hotplug') {
5029                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5030             } elsif ($opt eq 'tablet') {
5031                 die "skip\n" if !$hotplug_features->{usb};
5032                 if ($defaults->{tablet}) {
5033                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5034                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5035                         if $arch eq 'aarch64';
5036                 } else {
5037                     vm_deviceunplug($vmid, $conf, 'tablet');
5038                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5039                 }
5040             } elsif ($opt =~ m/^usb(\d+)$/) {
5041                 my $index = $1;
5042                 die "skip\n" if !$usb_hotplug;
5043                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5044                 vm_deviceunplug($vmid, $conf, $opt);
5045             } elsif ($opt eq 'vcpus') {
5046                 die "skip\n" if !$hotplug_features->{cpu};
5047                 qemu_cpu_hotplug($vmid, $conf, undef);
5048             } elsif ($opt eq 'balloon') {
5049                 # enable balloon device is not hotpluggable
5050                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5051                 # here we reset the ballooning value to memory
5052                 my $balloon = $conf->{memory} || $defaults->{memory};
5053                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5054             } elsif ($fast_plug_option->{$opt}) {
5055                 # do nothing
5056             } elsif ($opt =~ m/^net(\d+)$/) {
5057                 die "skip\n" if !$hotplug_features->{network};
5058                 vm_deviceunplug($vmid, $conf, $opt);
5059             } elsif (is_valid_drivename($opt)) {
5060                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5061                 vm_deviceunplug($vmid, $conf, $opt);
5062                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5063             } elsif ($opt =~ m/^memory$/) {
5064                 die "skip\n" if !$hotplug_features->{memory};
5065                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5066             } elsif ($opt eq 'cpuunits') {
5067                 $cgroup->change_cpu_shares(undef);
5068             } elsif ($opt eq 'cpulimit') {
5069                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5070             } else {
5071                 die "skip\n";
5072             }
5073         };
5074         if (my $err = $@) {
5075             &$add_error($opt, $err) if $err ne "skip\n";
5076         } else {
5077             my $old = delete $conf->{$opt};
5078             $cloudinit_record_changed->($conf, $opt, $old, undef);
5079             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5080         }
5081     }
5082
5083     my $cloudinit_opt;
5084     foreach my $opt (keys %{$conf->{pending}}) {
5085         next if $selection && !$selection->{$opt};
5086         my $value = $conf->{pending}->{$opt};
5087         eval {
5088             if ($opt eq 'hotplug') {
5089                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5090             } elsif ($opt eq 'tablet') {
5091                 die "skip\n" if !$hotplug_features->{usb};
5092                 if ($value == 1) {
5093                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5094                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5095                         if $arch eq 'aarch64';
5096                 } elsif ($value == 0) {
5097                     vm_deviceunplug($vmid, $conf, 'tablet');
5098                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5099                 }
5100             } elsif ($opt =~ m/^usb(\d+)$/) {
5101                 my $index = $1;
5102                 die "skip\n" if !$usb_hotplug;
5103                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5104                 my $id = $opt;
5105                 if ($d->{host} eq 'spice')  {
5106                     $id = "usbredirdev$index";
5107                 }
5108                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5109             } elsif ($opt eq 'vcpus') {
5110                 die "skip\n" if !$hotplug_features->{cpu};
5111                 qemu_cpu_hotplug($vmid, $conf, $value);
5112             } elsif ($opt eq 'balloon') {
5113                 # enable/disable balloning device is not hotpluggable
5114                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5115                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5116                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5117
5118                 # allow manual ballooning if shares is set to zero
5119                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5120                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5121                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5122                 }
5123             } elsif ($opt =~ m/^net(\d+)$/) {
5124                 # some changes can be done without hotplug
5125                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5126                                     $vmid, $opt, $value, $arch, $machine_type);
5127             } elsif (is_valid_drivename($opt)) {
5128                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5129                 # some changes can be done without hotplug
5130                 my $drive = parse_drive($opt, $value);
5131                 if (drive_is_cloudinit($drive)) {
5132                     $cloudinit_opt = [$opt, $drive];
5133                     # apply all the other changes first, then generate the cloudinit disk
5134                     die "skip\n";
5135                 }
5136                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5137                                      $vmid, $opt, $value, $arch, $machine_type);
5138             } elsif ($opt =~ m/^memory$/) { #dimms
5139                 die "skip\n" if !$hotplug_features->{memory};
5140                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5141             } elsif ($opt eq 'cpuunits') {
5142                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5143                 $cgroup->change_cpu_shares($new_cpuunits);
5144             } elsif ($opt eq 'cpulimit') {
5145                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5146                 $cgroup->change_cpu_quota($cpulimit, 100000);
5147             } elsif ($opt eq 'agent') {
5148                 vmconfig_update_agent($conf, $opt, $value);
5149             } else {
5150                 die "skip\n";  # skip non-hot-pluggable options
5151             }
5152         };
5153         if (my $err = $@) {
5154             &$add_error($opt, $err) if $err ne "skip\n";
5155         } else {
5156             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5157             $conf->{$opt} = $value;
5158             delete $conf->{pending}->{$opt};
5159         }
5160     }
5161
5162     if (defined($cloudinit_opt)) {
5163         my ($opt, $drive) = @$cloudinit_opt;
5164         my $value = $conf->{pending}->{$opt};
5165         eval {
5166             my $temp = {%$conf, $opt => $value};
5167             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5168             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5169                                  $vmid, $opt, $value, $arch, $machine_type);
5170         };
5171         if (my $err = $@) {
5172             &$add_error($opt, $err) if $err ne "skip\n";
5173         } else {
5174             $conf->{$opt} = $value;
5175             delete $conf->{pending}->{$opt};
5176         }
5177     }
5178
5179     # unplug xhci controller if no usb device is left
5180     if ($usb_hotplug) {
5181         my $has_usb = 0;
5182         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5183             next if !defined($conf->{"usb$i"});
5184             $has_usb = 1;
5185             last;
5186         }
5187         if (!$has_usb) {
5188             vm_deviceunplug($vmid, $conf, 'xhci');
5189         }
5190     }
5191
5192     PVE::QemuConfig->write_config($vmid, $conf);
5193
5194     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5195         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5196     }
5197 }
5198
5199 sub try_deallocate_drive {
5200     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5201
5202     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5203         my $volid = $drive->{file};
5204         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5205             my $sid = PVE::Storage::parse_volume_id($volid);
5206             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5207
5208             # check if the disk is really unused
5209             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5210                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5211             PVE::Storage::vdisk_free($storecfg, $volid);
5212             return 1;
5213         } else {
5214             # If vm is not owner of this disk remove from config
5215             return 1;
5216         }
5217     }
5218
5219     return;
5220 }
5221
5222 sub vmconfig_delete_or_detach_drive {
5223     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5224
5225     my $drive = parse_drive($opt, $conf->{$opt});
5226
5227     my $rpcenv = PVE::RPCEnvironment::get();
5228     my $authuser = $rpcenv->get_user();
5229
5230     if ($force) {
5231         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5232         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5233     } else {
5234         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5235     }
5236 }
5237
5238
5239
5240 sub vmconfig_apply_pending {
5241     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5242
5243     return if !scalar(keys %{$conf->{pending}});
5244
5245     my $add_apply_error = sub {
5246         my ($opt, $msg) = @_;
5247         my $err_msg = "unable to apply pending change $opt : $msg";
5248         $errors->{$opt} = $err_msg;
5249         warn $err_msg;
5250     };
5251
5252     # cold plug
5253
5254     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5255     foreach my $opt (sort keys %$pending_delete_hash) {
5256         my $force = $pending_delete_hash->{$opt}->{force};
5257         eval {
5258             if ($opt =~ m/^unused/) {
5259                 die "internal error";
5260             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5261                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5262             }
5263         };
5264         if (my $err = $@) {
5265             $add_apply_error->($opt, $err);
5266         } else {
5267             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5268             delete $conf->{$opt};
5269         }
5270     }
5271
5272     PVE::QemuConfig->cleanup_pending($conf);
5273
5274     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5275
5276     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5277         next if $opt eq 'delete'; # just to be sure
5278         eval {
5279             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5280                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5281             }
5282         };
5283         if (my $err = $@) {
5284             $add_apply_error->($opt, $err);
5285         } else {
5286
5287             if (is_valid_drivename($opt)) {
5288                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5289                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5290             }
5291
5292             $conf->{$opt} = delete $conf->{pending}->{$opt};
5293         }
5294     }
5295
5296     # write all changes at once to avoid unnecessary i/o
5297     PVE::QemuConfig->write_config($vmid, $conf);
5298     if ($generate_cloudinit) {
5299         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5300             # After successful generation and if there were changes to be applied, update the
5301             # config to drop the {cloudinit} entry.
5302             PVE::QemuConfig->write_config($vmid, $conf);
5303         }
5304     }
5305 }
5306
5307 sub vmconfig_update_net {
5308     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5309
5310     my $newnet = parse_net($value);
5311
5312     if ($conf->{$opt}) {
5313         my $oldnet = parse_net($conf->{$opt});
5314
5315         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5316             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5317             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5318             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5319
5320             # for non online change, we try to hot-unplug
5321             die "skip\n" if !$hotplug;
5322             vm_deviceunplug($vmid, $conf, $opt);
5323         } else {
5324
5325             die "internal error" if $opt !~ m/net(\d+)/;
5326             my $iface = "tap${vmid}i$1";
5327
5328             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5329                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5330                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5331                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5332                 PVE::Network::tap_unplug($iface);
5333
5334                 if ($have_sdn) {
5335                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5336                 } else {
5337                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5338                 }
5339             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5340                 # Rate can be applied on its own but any change above needs to
5341                 # include the rate in tap_plug since OVS resets everything.
5342                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5343             }
5344
5345             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5346                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5347             }
5348
5349             return 1;
5350         }
5351     }
5352
5353     if ($hotplug) {
5354         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5355     } else {
5356         die "skip\n";
5357     }
5358 }
5359
5360 sub vmconfig_update_agent {
5361     my ($conf, $opt, $value) = @_;
5362
5363     die "skip\n" if !$conf->{$opt};
5364
5365     my $hotplug_options = { fstrim_cloned_disks => 1 };
5366
5367     my $old_agent = parse_guest_agent($conf);
5368     my $agent = parse_guest_agent({$opt => $value});
5369
5370     for my $option (keys %$agent) { # added/changed options
5371         next if defined($hotplug_options->{$option});
5372         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5373     }
5374
5375     for my $option (keys %$old_agent) { # removed options
5376         next if defined($hotplug_options->{$option});
5377         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5378     }
5379
5380     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5381 }
5382
5383 sub vmconfig_update_disk {
5384     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5385
5386     my $drive = parse_drive($opt, $value);
5387
5388     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5389         my $media = $drive->{media} || 'disk';
5390         my $oldmedia = $old_drive->{media} || 'disk';
5391         die "unable to change media type\n" if $media ne $oldmedia;
5392
5393         if (!drive_is_cdrom($old_drive)) {
5394
5395             if ($drive->{file} ne $old_drive->{file}) {
5396
5397                 die "skip\n" if !$hotplug;
5398
5399                 # unplug and register as unused
5400                 vm_deviceunplug($vmid, $conf, $opt);
5401                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5402
5403             } else {
5404                 # update existing disk
5405
5406                 # skip non hotpluggable value
5407                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5408                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5409                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5410                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5411                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5412                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5413                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5414                     die "skip\n";
5415                 }
5416
5417                 # apply throttle
5418                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5419                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5420                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5421                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5422                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5423                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5424                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5425                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5426                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5427                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5428                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5429                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5430                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5431                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5432                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5433                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5434                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5435                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5436
5437                     qemu_block_set_io_throttle(
5438                         $vmid,"drive-$opt",
5439                         ($drive->{mbps} || 0)*1024*1024,
5440                         ($drive->{mbps_rd} || 0)*1024*1024,
5441                         ($drive->{mbps_wr} || 0)*1024*1024,
5442                          $drive->{iops} || 0,
5443                          $drive->{iops_rd} || 0,
5444                          $drive->{iops_wr} || 0,
5445                         ($drive->{mbps_max} || 0)*1024*1024,
5446                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5447                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5448                          $drive->{iops_max} || 0,
5449                          $drive->{iops_rd_max} || 0,
5450                          $drive->{iops_wr_max} || 0,
5451                          $drive->{bps_max_length} || 1,
5452                          $drive->{bps_rd_max_length} || 1,
5453                          $drive->{bps_wr_max_length} || 1,
5454                          $drive->{iops_max_length} || 1,
5455                          $drive->{iops_rd_max_length} || 1,
5456                          $drive->{iops_wr_max_length} || 1,
5457                     );
5458
5459                 }
5460
5461                 return 1;
5462             }
5463
5464         } else { # cdrom
5465
5466             if ($drive->{file} eq 'none') {
5467                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5468                 if (drive_is_cloudinit($old_drive)) {
5469                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5470                 }
5471             } else {
5472                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5473
5474                 # force eject if locked
5475                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5476
5477                 if ($path) {
5478                     mon_cmd($vmid, "blockdev-change-medium",
5479                         id => "$opt", filename => "$path");
5480                 }
5481             }
5482
5483             return 1;
5484         }
5485     }
5486
5487     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5488     # hotplug new disks
5489     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5490     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5491 }
5492
5493 sub vmconfig_update_cloudinit_drive {
5494     my ($storecfg, $conf, $vmid) = @_;
5495
5496     my $cloudinit_ds = undef;
5497     my $cloudinit_drive = undef;
5498
5499     PVE::QemuConfig->foreach_volume($conf, sub {
5500         my ($ds, $drive) = @_;
5501         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5502             $cloudinit_ds = $ds;
5503             $cloudinit_drive = $drive;
5504         }
5505     });
5506
5507     return if !$cloudinit_drive;
5508
5509     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5510         PVE::QemuConfig->write_config($vmid, $conf);
5511     }
5512
5513     my $running = PVE::QemuServer::check_running($vmid);
5514
5515     if ($running) {
5516         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5517         if ($path) {
5518             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5519             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5520         }
5521     }
5522 }
5523
5524 # called in locked context by incoming migration
5525 sub vm_migrate_get_nbd_disks {
5526     my ($storecfg, $conf, $replicated_volumes) = @_;
5527
5528     my $local_volumes = {};
5529     PVE::QemuConfig->foreach_volume($conf, sub {
5530         my ($ds, $drive) = @_;
5531
5532         return if drive_is_cdrom($drive);
5533         return if $ds eq 'tpmstate0';
5534
5535         my $volid = $drive->{file};
5536
5537         return if !$volid;
5538
5539         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5540
5541         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5542         return if $scfg->{shared};
5543
5544         # replicated disks re-use existing state via bitmap
5545         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5546         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5547     });
5548     return $local_volumes;
5549 }
5550
5551 # called in locked context by incoming migration
5552 sub vm_migrate_alloc_nbd_disks {
5553     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5554
5555     my $nbd = {};
5556     foreach my $opt (sort keys %$source_volumes) {
5557         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5558
5559         if ($use_existing) {
5560             $nbd->{$opt}->{drivestr} = print_drive($drive);
5561             $nbd->{$opt}->{volid} = $volid;
5562             $nbd->{$opt}->{replicated} = 1;
5563             next;
5564         }
5565
5566         # storage mapping + volname = regular migration
5567         # storage mapping + format = remote migration
5568         # order of precedence, filtered by whether storage supports it:
5569         # 1. explicit requested format
5570         # 2. format of current volume
5571         # 3. default format of storage
5572         if (!$storagemap->{identity}) {
5573             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5574             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5575             if (!$format || !grep { $format eq $_ } @$validFormats) {
5576                 if ($volname) {
5577                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5578                     my $fileFormat = qemu_img_format($scfg, $volname);
5579                     $format = $fileFormat
5580                         if grep { $fileFormat eq $_ } @$validFormats;
5581                 }
5582                 $format //= $defFormat;
5583             }
5584         } else {
5585             # can't happen for remote migration, so $volname is always defined
5586             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5587             $format = qemu_img_format($scfg, $volname);
5588         }
5589
5590         my $size = $drive->{size} / 1024;
5591         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5592         my $newdrive = $drive;
5593         $newdrive->{format} = $format;
5594         $newdrive->{file} = $newvolid;
5595         my $drivestr = print_drive($newdrive);
5596         $nbd->{$opt}->{drivestr} = $drivestr;
5597         $nbd->{$opt}->{volid} = $newvolid;
5598     }
5599
5600     return $nbd;
5601 }
5602
5603 # see vm_start_nolock for parameters, additionally:
5604 # migrate_opts:
5605 #   storagemap = parsed storage map for allocating NBD disks
5606 sub vm_start {
5607     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5608
5609     return PVE::QemuConfig->lock_config($vmid, sub {
5610         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5611
5612         die "you can't start a vm if it's a template\n"
5613             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5614
5615         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5616         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5617
5618         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5619
5620         if ($has_backup_lock && $running) {
5621             # a backup is currently running, attempt to start the guest in the
5622             # existing QEMU instance
5623             return vm_resume($vmid);
5624         }
5625
5626         PVE::QemuConfig->check_lock($conf)
5627             if !($params->{skiplock} || $has_suspended_lock);
5628
5629         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5630
5631         die "VM $vmid already running\n" if $running;
5632
5633         if (my $storagemap = $migrate_opts->{storagemap}) {
5634             my $replicated = $migrate_opts->{replicated_volumes};
5635             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5636             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5637
5638             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5639                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5640             }
5641         }
5642
5643         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5644     });
5645 }
5646
5647
5648 # params:
5649 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5650 #   skiplock => 0/1, skip checking for config lock
5651 #   skiptemplate => 0/1, skip checking whether VM is template
5652 #   forcemachine => to force QEMU machine (rollback/migration)
5653 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5654 #   timeout => in seconds
5655 #   paused => start VM in paused state (backup)
5656 #   resume => resume from hibernation
5657 #   pbs-backing => {
5658 #      sata0 => {
5659 #         repository
5660 #         snapshot
5661 #         keyfile
5662 #         archive
5663 #      },
5664 #      virtio2 => ...
5665 #   }
5666 # migrate_opts:
5667 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5668 #   migratedfrom => source node
5669 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5670 #   network => CIDR of migration network
5671 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5672 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5673 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5674 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5675 #       contained in config
5676 sub vm_start_nolock {
5677     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5678
5679     my $statefile = $params->{statefile};
5680     my $resume = $params->{resume};
5681
5682     my $migratedfrom = $migrate_opts->{migratedfrom};
5683     my $migration_type = $migrate_opts->{type};
5684
5685     my $res = {};
5686
5687     # clean up leftover reboot request files
5688     eval { clear_reboot_request($vmid); };
5689     warn $@ if $@;
5690
5691     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5692         vmconfig_apply_pending($vmid, $conf, $storecfg);
5693         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5694     }
5695
5696     # don't regenerate the ISO if the VM is started as part of a live migration
5697     # this way we can reuse the old ISO with the correct config
5698     if (!$migratedfrom) {
5699         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5700             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5701             # $conf->{cloudinit}, so we could just not do this?
5702             # But we do it above, so for now let's be consistent.
5703             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5704         }
5705     }
5706
5707     # override offline migrated volumes, conf is out of date still
5708     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5709         for my $key (sort keys $offline_volumes->%*) {
5710             my $parsed = parse_drive($key, $conf->{$key});
5711             $parsed->{file} = $offline_volumes->{$key};
5712             $conf->{$key} = print_drive($parsed);
5713         }
5714     }
5715
5716     my $defaults = load_defaults();
5717
5718     # set environment variable useful inside network script
5719     # for remote migration the config is available on the target node!
5720     if (!$migrate_opts->{remote_node}) {
5721         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5722     }
5723
5724     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5725
5726     my $forcemachine = $params->{forcemachine};
5727     my $forcecpu = $params->{forcecpu};
5728     if ($resume) {
5729         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5730         $forcemachine = $conf->{runningmachine};
5731         $forcecpu = $conf->{runningcpu};
5732         print "Resuming suspended VM\n";
5733     }
5734
5735     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5736         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5737
5738     my $migration_ip;
5739     my $get_migration_ip = sub {
5740         my ($nodename) = @_;
5741
5742         return $migration_ip if defined($migration_ip);
5743
5744         my $cidr = $migrate_opts->{network};
5745
5746         if (!defined($cidr)) {
5747             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5748             $cidr = $dc_conf->{migration}->{network};
5749         }
5750
5751         if (defined($cidr)) {
5752             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5753
5754             die "could not get IP: no address configured on local " .
5755                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5756
5757             die "could not get IP: multiple addresses configured on local " .
5758                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5759
5760             $migration_ip = @$ips[0];
5761         }
5762
5763         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5764             if !defined($migration_ip);
5765
5766         return $migration_ip;
5767     };
5768
5769     if ($statefile) {
5770         if ($statefile eq 'tcp') {
5771             my $migrate = $res->{migrate} = { proto => 'tcp' };
5772             $migrate->{addr} = "localhost";
5773             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5774             my $nodename = nodename();
5775
5776             if (!defined($migration_type)) {
5777                 if (defined($datacenterconf->{migration}->{type})) {
5778                     $migration_type = $datacenterconf->{migration}->{type};
5779                 } else {
5780                     $migration_type = 'secure';
5781                 }
5782             }
5783
5784             if ($migration_type eq 'insecure') {
5785                 $migrate->{addr} = $get_migration_ip->($nodename);
5786                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5787             }
5788
5789             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5790             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5791             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5792             push @$cmd, '-incoming', $migrate->{uri};
5793             push @$cmd, '-S';
5794
5795         } elsif ($statefile eq 'unix') {
5796             # should be default for secure migrations as a ssh TCP forward
5797             # tunnel is not deterministic reliable ready and fails regurarly
5798             # to set up in time, so use UNIX socket forwards
5799             my $migrate = $res->{migrate} = { proto => 'unix' };
5800             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5801             unlink $migrate->{addr};
5802
5803             $migrate->{uri} = "unix:$migrate->{addr}";
5804             push @$cmd, '-incoming', $migrate->{uri};
5805             push @$cmd, '-S';
5806
5807         } elsif (-e $statefile) {
5808             push @$cmd, '-loadstate', $statefile;
5809         } else {
5810             my $statepath = PVE::Storage::path($storecfg, $statefile);
5811             push @$vollist, $statefile;
5812             push @$cmd, '-loadstate', $statepath;
5813         }
5814     } elsif ($params->{paused}) {
5815         push @$cmd, '-S';
5816     }
5817
5818     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5819
5820     my $pci_devices = {}; # host pci devices
5821     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5822         my $dev = $conf->{"hostpci$i"} or next;
5823         $pci_devices->{$i} = parse_hostpci($dev);
5824     }
5825
5826     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5827     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5828
5829     # map to a flat list of pci ids
5830     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5831
5832     # reserve all PCI IDs before actually doing anything with them
5833     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5834
5835     eval {
5836         my $uuid;
5837         for my $id (sort keys %$pci_devices) {
5838             my $d = $pci_devices->{$id};
5839             for my $dev ($d->{pciid}->@*) {
5840                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5841
5842                 # nvidia grid needs the uuid of the mdev as qemu parameter
5843                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5844                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5845                 }
5846             }
5847         }
5848         push @$cmd, '-uuid', $uuid if defined($uuid);
5849     };
5850     if (my $err = $@) {
5851         eval { cleanup_pci_devices($vmid, $conf) };
5852         warn $@ if $@;
5853         die $err;
5854     }
5855
5856     PVE::Storage::activate_volumes($storecfg, $vollist);
5857
5858     eval {
5859         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5860     };
5861     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5862     # timeout should be more than enough here...
5863     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5864
5865     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5866
5867     my %run_params = (
5868         timeout => $statefile ? undef : $start_timeout,
5869         umask => 0077,
5870         noerr => 1,
5871     );
5872
5873     # when migrating, prefix QEMU output so other side can pick up any
5874     # errors that might occur and show the user
5875     if ($migratedfrom) {
5876         $run_params{quiet} = 1;
5877         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5878     }
5879
5880     my %systemd_properties = (
5881         Slice => 'qemu.slice',
5882         KillMode => 'process',
5883         SendSIGKILL => 0,
5884         TimeoutStopUSec => ULONG_MAX, # infinity
5885     );
5886
5887     if (PVE::CGroup::cgroup_mode() == 2) {
5888         $systemd_properties{CPUWeight} = $cpuunits;
5889     } else {
5890         $systemd_properties{CPUShares} = $cpuunits;
5891     }
5892
5893     if (my $cpulimit = $conf->{cpulimit}) {
5894         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5895     }
5896     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5897
5898     my $run_qemu = sub {
5899         PVE::Tools::run_fork sub {
5900             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5901
5902             my $tpmpid;
5903             if (my $tpm = $conf->{tpmstate0}) {
5904                 # start the TPM emulator so QEMU can connect on start
5905                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5906             }
5907
5908             my $exitcode = run_command($cmd, %run_params);
5909             if ($exitcode) {
5910                 if ($tpmpid) {
5911                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5912                     kill 'TERM', $tpmpid;
5913                 }
5914                 die "QEMU exited with code $exitcode\n";
5915             }
5916         };
5917     };
5918
5919     if ($conf->{hugepages}) {
5920
5921         my $code = sub {
5922             my $hotplug_features =
5923                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5924             my $hugepages_topology =
5925                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5926
5927             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5928
5929             PVE::QemuServer::Memory::hugepages_mount();
5930             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5931
5932             eval { $run_qemu->() };
5933             if (my $err = $@) {
5934                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5935                     if !$conf->{keephugepages};
5936                 die $err;
5937             }
5938
5939             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5940                 if !$conf->{keephugepages};
5941         };
5942         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5943
5944     } else {
5945         eval { $run_qemu->() };
5946     }
5947
5948     if (my $err = $@) {
5949         # deactivate volumes if start fails
5950         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5951         warn $@ if $@;
5952         eval { cleanup_pci_devices($vmid, $conf) };
5953         warn $@ if $@;
5954
5955         die "start failed: $err";
5956     }
5957
5958     # re-reserve all PCI IDs now that we can know the actual VM PID
5959     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5960     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5961     warn $@ if $@;
5962
5963     if (defined($res->{migrate})) {
5964         print "migration listens on $res->{migrate}->{uri}\n";
5965     } elsif ($statefile) {
5966         eval { mon_cmd($vmid, "cont"); };
5967         warn $@ if $@;
5968     }
5969
5970     #start nbd server for storage migration
5971     if (my $nbd = $migrate_opts->{nbd}) {
5972         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5973
5974         my $migrate_storage_uri;
5975         # nbd_protocol_version > 0 for unix socket support
5976         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5977             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5978             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5979             $migrate_storage_uri = "nbd:unix:$socket_path";
5980             $res->{migrate}->{unix_sockets} = [$socket_path];
5981         } else {
5982             my $nodename = nodename();
5983             my $localip = $get_migration_ip->($nodename);
5984             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5985             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5986
5987             mon_cmd($vmid, "nbd-server-start", addr => {
5988                 type => 'inet',
5989                 data => {
5990                     host => "${localip}",
5991                     port => "${storage_migrate_port}",
5992                 },
5993             });
5994             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5995             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5996         }
5997
5998         my $block_info = mon_cmd($vmid, "query-block");
5999         $block_info = { map { $_->{device} => $_ } $block_info->@* };
6000
6001         foreach my $opt (sort keys %$nbd) {
6002             my $drivestr = $nbd->{$opt}->{drivestr};
6003             my $volid = $nbd->{$opt}->{volid};
6004
6005             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
6006
6007             mon_cmd(
6008                 $vmid,
6009                 "block-export-add",
6010                 id => "drive-$opt",
6011                 'node-name' => $block_node,
6012                 writable => JSON::true,
6013                 type => "nbd",
6014                 name => "drive-$opt", # NBD export name
6015             );
6016
6017             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6018             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6019             print "re-using replicated volume: $opt - $volid\n"
6020                 if $nbd->{$opt}->{replicated};
6021
6022             $res->{drives}->{$opt} = $nbd->{$opt};
6023             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6024         }
6025     }
6026
6027     if ($migratedfrom) {
6028         eval {
6029             set_migration_caps($vmid);
6030         };
6031         warn $@ if $@;
6032
6033         if ($spice_port) {
6034             print "spice listens on port $spice_port\n";
6035             $res->{spice_port} = $spice_port;
6036             if ($migrate_opts->{spice_ticket}) {
6037                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6038                     $migrate_opts->{spice_ticket});
6039                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6040             }
6041         }
6042
6043     } else {
6044         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6045             if !$statefile && $conf->{balloon};
6046
6047         foreach my $opt (keys %$conf) {
6048             next if $opt !~  m/^net\d+$/;
6049             my $nicconf = parse_net($conf->{$opt});
6050             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6051         }
6052         add_nets_bridge_fdb($conf, $vmid);
6053     }
6054
6055    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6056         eval {
6057             mon_cmd(
6058                 $vmid,
6059                 'qom-set',
6060                 path => "machine/peripheral/balloon0",
6061                 property => "guest-stats-polling-interval",
6062                 value => 2
6063             );
6064         };
6065         log_warn("could not set polling interval for ballooning - $@") if $@;
6066     }
6067
6068     if ($resume) {
6069         print "Resumed VM, removing state\n";
6070         if (my $vmstate = $conf->{vmstate}) {
6071             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6072             PVE::Storage::vdisk_free($storecfg, $vmstate);
6073         }
6074         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6075         PVE::QemuConfig->write_config($vmid, $conf);
6076     }
6077
6078     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6079
6080     return $res;
6081 }
6082
6083 sub vm_commandline {
6084     my ($storecfg, $vmid, $snapname) = @_;
6085
6086     my $conf = PVE::QemuConfig->load_config($vmid);
6087
6088     my ($forcemachine, $forcecpu);
6089     if ($snapname) {
6090         my $snapshot = $conf->{snapshots}->{$snapname};
6091         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6092
6093         # check for machine or CPU overrides in snapshot
6094         $forcemachine = $snapshot->{runningmachine};
6095         $forcecpu = $snapshot->{runningcpu};
6096
6097         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6098
6099         $conf = $snapshot;
6100     }
6101
6102     my $defaults = load_defaults();
6103
6104     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6105
6106     return PVE::Tools::cmd2string($cmd);
6107 }
6108
6109 sub vm_reset {
6110     my ($vmid, $skiplock) = @_;
6111
6112     PVE::QemuConfig->lock_config($vmid, sub {
6113
6114         my $conf = PVE::QemuConfig->load_config($vmid);
6115
6116         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6117
6118         mon_cmd($vmid, "system_reset");
6119     });
6120 }
6121
6122 sub get_vm_volumes {
6123     my ($conf) = @_;
6124
6125     my $vollist = [];
6126     foreach_volid($conf, sub {
6127         my ($volid, $attr) = @_;
6128
6129         return if $volid =~ m|^/|;
6130
6131         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6132         return if !$sid;
6133
6134         push @$vollist, $volid;
6135     });
6136
6137     return $vollist;
6138 }
6139
6140 sub cleanup_pci_devices {
6141     my ($vmid, $conf) = @_;
6142
6143     foreach my $key (keys %$conf) {
6144         next if $key !~ m/^hostpci(\d+)$/;
6145         my $hostpciindex = $1;
6146         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6147         my $d = parse_hostpci($conf->{$key});
6148         if ($d->{mdev}) {
6149             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6150             # don't want to break ABI just for this two liner
6151             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6152             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6153         }
6154     }
6155     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6156 }
6157
6158 sub vm_stop_cleanup {
6159     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6160
6161     eval {
6162
6163         if (!$keepActive) {
6164             my $vollist = get_vm_volumes($conf);
6165             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6166
6167             if (my $tpmdrive = $conf->{tpmstate0}) {
6168                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6169                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6170                 if ($storeid) {
6171                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6172                 }
6173             }
6174         }
6175
6176         foreach my $ext (qw(mon qmp pid vnc qga)) {
6177             unlink "/var/run/qemu-server/${vmid}.$ext";
6178         }
6179
6180         if ($conf->{ivshmem}) {
6181             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6182             # just delete it for now, VMs which have this already open do not
6183             # are affected, but new VMs will get a separated one. If this
6184             # becomes an issue we either add some sort of ref-counting or just
6185             # add a "don't delete on stop" flag to the ivshmem format.
6186             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6187         }
6188
6189         cleanup_pci_devices($vmid, $conf);
6190
6191         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6192     };
6193     warn $@ if $@; # avoid errors - just warn
6194 }
6195
6196 # call only in locked context
6197 sub _do_vm_stop {
6198     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6199
6200     my $pid = check_running($vmid, $nocheck);
6201     return if !$pid;
6202
6203     my $conf;
6204     if (!$nocheck) {
6205         $conf = PVE::QemuConfig->load_config($vmid);
6206         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6207         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6208             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6209             $timeout = $opts->{down} if $opts->{down};
6210         }
6211         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6212     }
6213
6214     eval {
6215         if ($shutdown) {
6216             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6217                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6218             } else {
6219                 mon_cmd($vmid, "system_powerdown");
6220             }
6221         } else {
6222             mon_cmd($vmid, "quit");
6223         }
6224     };
6225     my $err = $@;
6226
6227     if (!$err) {
6228         $timeout = 60 if !defined($timeout);
6229
6230         my $count = 0;
6231         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6232             $count++;
6233             sleep 1;
6234         }
6235
6236         if ($count >= $timeout) {
6237             if ($force) {
6238                 warn "VM still running - terminating now with SIGTERM\n";
6239                 kill 15, $pid;
6240             } else {
6241                 die "VM quit/powerdown failed - got timeout\n";
6242             }
6243         } else {
6244             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6245             return;
6246         }
6247     } else {
6248        if (!check_running($vmid, $nocheck)) {
6249             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6250             return;
6251        }
6252        if ($force) {
6253             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6254             kill 15, $pid;
6255         } else {
6256             die "VM quit/powerdown failed\n";
6257         }
6258     }
6259
6260     # wait again
6261     $timeout = 10;
6262
6263     my $count = 0;
6264     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6265         $count++;
6266         sleep 1;
6267     }
6268
6269     if ($count >= $timeout) {
6270         warn "VM still running - terminating now with SIGKILL\n";
6271         kill 9, $pid;
6272         sleep 1;
6273     }
6274
6275     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6276 }
6277
6278 # Note: use $nocheck to skip tests if VM configuration file exists.
6279 # We need that when migration VMs to other nodes (files already moved)
6280 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6281 sub vm_stop {
6282     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6283
6284     $force = 1 if !defined($force) && !$shutdown;
6285
6286     if ($migratedfrom){
6287         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6288         kill 15, $pid if $pid;
6289         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6290         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6291         return;
6292     }
6293
6294     PVE::QemuConfig->lock_config($vmid, sub {
6295         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6296    });
6297 }
6298
6299 sub vm_reboot {
6300     my ($vmid, $timeout) = @_;
6301
6302     PVE::QemuConfig->lock_config($vmid, sub {
6303         eval {
6304
6305             # only reboot if running, as qmeventd starts it again on a stop event
6306             return if !check_running($vmid);
6307
6308             create_reboot_request($vmid);
6309
6310             my $storecfg = PVE::Storage::config();
6311             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6312
6313         };
6314         if (my $err = $@) {
6315             # avoid that the next normal shutdown will be confused for a reboot
6316             clear_reboot_request($vmid);
6317             die $err;
6318         }
6319    });
6320 }
6321
6322 # note: if using the statestorage parameter, the caller has to check privileges
6323 sub vm_suspend {
6324     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6325
6326     my $conf;
6327     my $path;
6328     my $storecfg;
6329     my $vmstate;
6330
6331     PVE::QemuConfig->lock_config($vmid, sub {
6332
6333         $conf = PVE::QemuConfig->load_config($vmid);
6334
6335         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6336         PVE::QemuConfig->check_lock($conf)
6337             if !($skiplock || $is_backing_up);
6338
6339         die "cannot suspend to disk during backup\n"
6340             if $is_backing_up && $includestate;
6341
6342         if ($includestate) {
6343             $conf->{lock} = 'suspending';
6344             my $date = strftime("%Y-%m-%d", localtime(time()));
6345             $storecfg = PVE::Storage::config();
6346             if (!$statestorage) {
6347                 $statestorage = find_vmstate_storage($conf, $storecfg);
6348                 # check permissions for the storage
6349                 my $rpcenv = PVE::RPCEnvironment::get();
6350                 if ($rpcenv->{type} ne 'cli') {
6351                     my $authuser = $rpcenv->get_user();
6352                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6353                 }
6354             }
6355
6356
6357             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6358                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6359             $path = PVE::Storage::path($storecfg, $vmstate);
6360             PVE::QemuConfig->write_config($vmid, $conf);
6361         } else {
6362             mon_cmd($vmid, "stop");
6363         }
6364     });
6365
6366     if ($includestate) {
6367         # save vm state
6368         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6369
6370         eval {
6371             set_migration_caps($vmid, 1);
6372             mon_cmd($vmid, "savevm-start", statefile => $path);
6373             for(;;) {
6374                 my $state = mon_cmd($vmid, "query-savevm");
6375                 if (!$state->{status}) {
6376                     die "savevm not active\n";
6377                 } elsif ($state->{status} eq 'active') {
6378                     sleep(1);
6379                     next;
6380                 } elsif ($state->{status} eq 'completed') {
6381                     print "State saved, quitting\n";
6382                     last;
6383                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6384                     die "query-savevm failed with error '$state->{error}'\n"
6385                 } else {
6386                     die "query-savevm returned status '$state->{status}'\n";
6387                 }
6388             }
6389         };
6390         my $err = $@;
6391
6392         PVE::QemuConfig->lock_config($vmid, sub {
6393             $conf = PVE::QemuConfig->load_config($vmid);
6394             if ($err) {
6395                 # cleanup, but leave suspending lock, to indicate something went wrong
6396                 eval {
6397                     mon_cmd($vmid, "savevm-end");
6398                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6399                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6400                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6401                     PVE::QemuConfig->write_config($vmid, $conf);
6402                 };
6403                 warn $@ if $@;
6404                 die $err;
6405             }
6406
6407             die "lock changed unexpectedly\n"
6408                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6409
6410             mon_cmd($vmid, "quit");
6411             $conf->{lock} = 'suspended';
6412             PVE::QemuConfig->write_config($vmid, $conf);
6413         });
6414     }
6415 }
6416
6417 # $nocheck is set when called as part of a migration - in this context the
6418 # location of the config file (source or target node) is not deterministic,
6419 # since migration cannot wait for pmxcfs to process the rename
6420 sub vm_resume {
6421     my ($vmid, $skiplock, $nocheck) = @_;
6422
6423     PVE::QemuConfig->lock_config($vmid, sub {
6424         my $res = mon_cmd($vmid, 'query-status');
6425         my $resume_cmd = 'cont';
6426         my $reset = 0;
6427         my $conf;
6428         if ($nocheck) {
6429             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6430             if ($@) {
6431                 my $vmlist = PVE::Cluster::get_vmlist();
6432                 if (exists($vmlist->{ids}->{$vmid})) {
6433                     my $node = $vmlist->{ids}->{$vmid}->{node};
6434                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6435                 }
6436                 if (!$conf) {
6437                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6438                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6439                 }
6440             }
6441         } else {
6442             $conf = PVE::QemuConfig->load_config($vmid);
6443         }
6444
6445         if ($res->{status}) {
6446             return if $res->{status} eq 'running'; # job done, go home
6447             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6448             $reset = 1 if $res->{status} eq 'shutdown';
6449         }
6450
6451         if (!$nocheck) {
6452             PVE::QemuConfig->check_lock($conf)
6453                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6454         }
6455
6456         if ($reset) {
6457             # required if a VM shuts down during a backup and we get a resume
6458             # request before the backup finishes for example
6459             mon_cmd($vmid, "system_reset");
6460         }
6461
6462         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6463
6464         mon_cmd($vmid, $resume_cmd);
6465     });
6466 }
6467
6468 sub vm_sendkey {
6469     my ($vmid, $skiplock, $key) = @_;
6470
6471     PVE::QemuConfig->lock_config($vmid, sub {
6472
6473         my $conf = PVE::QemuConfig->load_config($vmid);
6474
6475         # there is no qmp command, so we use the human monitor command
6476         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6477         die $res if $res ne '';
6478     });
6479 }
6480
6481 # vzdump restore implementaion
6482
6483 sub tar_archive_read_firstfile {
6484     my $archive = shift;
6485
6486     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6487
6488     # try to detect archive type first
6489     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6490         die "unable to open file '$archive'\n";
6491     my $firstfile = <$fh>;
6492     kill 15, $pid;
6493     close $fh;
6494
6495     die "ERROR: archive contaions no data\n" if !$firstfile;
6496     chomp $firstfile;
6497
6498     return $firstfile;
6499 }
6500
6501 sub tar_restore_cleanup {
6502     my ($storecfg, $statfile) = @_;
6503
6504     print STDERR "starting cleanup\n";
6505
6506     if (my $fd = IO::File->new($statfile, "r")) {
6507         while (defined(my $line = <$fd>)) {
6508             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6509                 my $volid = $2;
6510                 eval {
6511                     if ($volid =~ m|^/|) {
6512                         unlink $volid || die 'unlink failed\n';
6513                     } else {
6514                         PVE::Storage::vdisk_free($storecfg, $volid);
6515                     }
6516                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6517                 };
6518                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6519             } else {
6520                 print STDERR "unable to parse line in statfile - $line";
6521             }
6522         }
6523         $fd->close();
6524     }
6525 }
6526
6527 sub restore_file_archive {
6528     my ($archive, $vmid, $user, $opts) = @_;
6529
6530     return restore_vma_archive($archive, $vmid, $user, $opts)
6531         if $archive eq '-';
6532
6533     my $info = PVE::Storage::archive_info($archive);
6534     my $format = $opts->{format} // $info->{format};
6535     my $comp = $info->{compression};
6536
6537     # try to detect archive format
6538     if ($format eq 'tar') {
6539         return restore_tar_archive($archive, $vmid, $user, $opts);
6540     } else {
6541         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6542     }
6543 }
6544
6545 # hepler to remove disks that will not be used after restore
6546 my $restore_cleanup_oldconf = sub {
6547     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6548
6549     my $kept_disks = {};
6550
6551     PVE::QemuConfig->foreach_volume($oldconf, sub {
6552         my ($ds, $drive) = @_;
6553
6554         return if drive_is_cdrom($drive, 1);
6555
6556         my $volid = $drive->{file};
6557         return if !$volid || $volid =~ m|^/|;
6558
6559         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6560         return if !$path || !$owner || ($owner != $vmid);
6561
6562         # Note: only delete disk we want to restore
6563         # other volumes will become unused
6564         if ($virtdev_hash->{$ds}) {
6565             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6566             if (my $err = $@) {
6567                 warn $err;
6568             }
6569         } else {
6570             $kept_disks->{$volid} = 1;
6571         }
6572     });
6573
6574     # after the restore we have no snapshots anymore
6575     for my $snapname (keys $oldconf->{snapshots}->%*) {
6576         my $snap = $oldconf->{snapshots}->{$snapname};
6577         if ($snap->{vmstate}) {
6578             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6579             if (my $err = $@) {
6580                 warn $err;
6581             }
6582         }
6583
6584         for my $volid (keys $kept_disks->%*) {
6585             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6586             warn $@ if $@;
6587         }
6588     }
6589 };
6590
6591 # Helper to parse vzdump backup device hints
6592 #
6593 # $rpcenv: Environment, used to ckeck storage permissions
6594 # $user: User ID, to check storage permissions
6595 # $storecfg: Storage configuration
6596 # $fh: the file handle for reading the configuration
6597 # $devinfo: should contain device sizes for all backu-up'ed devices
6598 # $options: backup options (pool, default storage)
6599 #
6600 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6601 my $parse_backup_hints = sub {
6602     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6603
6604     my $check_storage = sub { # assert if an image can be allocate
6605         my ($storeid, $scfg) = @_;
6606         die "Content type 'images' is not available on storage '$storeid'\n"
6607             if !$scfg->{content}->{images};
6608         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6609             if $user ne 'root@pam';
6610     };
6611
6612     my $virtdev_hash = {};
6613     while (defined(my $line = <$fh>)) {
6614         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6615             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6616             die "archive does not contain data for drive '$virtdev'\n"
6617                 if !$devinfo->{$devname};
6618
6619             if (defined($options->{storage})) {
6620                 $storeid = $options->{storage} || 'local';
6621             } elsif (!$storeid) {
6622                 $storeid = 'local';
6623             }
6624             $format = 'raw' if !$format;
6625             $devinfo->{$devname}->{devname} = $devname;
6626             $devinfo->{$devname}->{virtdev} = $virtdev;
6627             $devinfo->{$devname}->{format} = $format;
6628             $devinfo->{$devname}->{storeid} = $storeid;
6629
6630             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6631             $check_storage->($storeid, $scfg); # permission and content type check
6632
6633             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6634         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6635             my $virtdev = $1;
6636             my $drive = parse_drive($virtdev, $2);
6637
6638             if (drive_is_cloudinit($drive)) {
6639                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6640                 $storeid = $options->{storage} if defined ($options->{storage});
6641                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6642                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6643
6644                 $check_storage->($storeid, $scfg); # permission and content type check
6645
6646                 $virtdev_hash->{$virtdev} = {
6647                     format => $format,
6648                     storeid => $storeid,
6649                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6650                     is_cloudinit => 1,
6651                 };
6652             }
6653         }
6654     }
6655
6656     return $virtdev_hash;
6657 };
6658
6659 # Helper to allocate and activate all volumes required for a restore
6660 #
6661 # $storecfg: Storage configuration
6662 # $virtdev_hash: as returned by parse_backup_hints()
6663 #
6664 # Returns: { $virtdev => $volid }
6665 my $restore_allocate_devices = sub {
6666     my ($storecfg, $virtdev_hash, $vmid) = @_;
6667
6668     my $map = {};
6669     foreach my $virtdev (sort keys %$virtdev_hash) {
6670         my $d = $virtdev_hash->{$virtdev};
6671         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6672         my $storeid = $d->{storeid};
6673         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6674
6675         # test if requested format is supported
6676         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6677         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6678         $d->{format} = $defFormat if !$supported;
6679
6680         my $name;
6681         if ($d->{is_cloudinit}) {
6682             $name = "vm-$vmid-cloudinit";
6683             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6684             if ($scfg->{path}) {
6685                 $name .= ".$d->{format}";
6686             }
6687         }
6688
6689         my $volid = PVE::Storage::vdisk_alloc(
6690             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6691
6692         print STDERR "new volume ID is '$volid'\n";
6693         $d->{volid} = $volid;
6694
6695         PVE::Storage::activate_volumes($storecfg, [$volid]);
6696
6697         $map->{$virtdev} = $volid;
6698     }
6699
6700     return $map;
6701 };
6702
6703 sub restore_update_config_line {
6704     my ($cookie, $map, $line, $unique) = @_;
6705
6706     return '' if $line =~ m/^\#qmdump\#/;
6707     return '' if $line =~ m/^\#vzdump\#/;
6708     return '' if $line =~ m/^lock:/;
6709     return '' if $line =~ m/^unused\d+:/;
6710     return '' if $line =~ m/^parent:/;
6711
6712     my $res = '';
6713
6714     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6715     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6716         # try to convert old 1.X settings
6717         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6718         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6719             my ($model, $macaddr) = split(/\=/, $devconfig);
6720             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6721             my $net = {
6722                 model => $model,
6723                 bridge => "vmbr$ind",
6724                 macaddr => $macaddr,
6725             };
6726             my $netstr = print_net($net);
6727
6728             $res .= "net$cookie->{netcount}: $netstr\n";
6729             $cookie->{netcount}++;
6730         }
6731     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6732         my ($id, $netstr) = ($1, $2);
6733         my $net = parse_net($netstr);
6734         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6735         $netstr = print_net($net);
6736         $res .= "$id: $netstr\n";
6737     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6738         my $virtdev = $1;
6739         my $value = $3;
6740         my $di = parse_drive($virtdev, $value);
6741         if (defined($di->{backup}) && !$di->{backup}) {
6742             $res .= "#$line";
6743         } elsif ($map->{$virtdev}) {
6744             delete $di->{format}; # format can change on restore
6745             $di->{file} = $map->{$virtdev};
6746             $value = print_drive($di);
6747             $res .= "$virtdev: $value\n";
6748         } else {
6749             $res .= $line;
6750         }
6751     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6752         my $vmgenid = $1;
6753         if ($vmgenid ne '0') {
6754             # always generate a new vmgenid if there was a valid one setup
6755             $vmgenid = generate_uuid();
6756         }
6757         $res .= "vmgenid: $vmgenid\n";
6758     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6759         my ($uuid, $uuid_str);
6760         UUID::generate($uuid);
6761         UUID::unparse($uuid, $uuid_str);
6762         my $smbios1 = parse_smbios1($2);
6763         $smbios1->{uuid} = $uuid_str;
6764         $res .= $1.print_smbios1($smbios1)."\n";
6765     } else {
6766         $res .= $line;
6767     }
6768
6769     return $res;
6770 }
6771
6772 my $restore_deactivate_volumes = sub {
6773     my ($storecfg, $virtdev_hash) = @_;
6774
6775     my $vollist = [];
6776     for my $dev (values $virtdev_hash->%*) {
6777         push $vollist->@*, $dev->{volid} if $dev->{volid};
6778     }
6779
6780     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6781     print STDERR $@ if $@;
6782 };
6783
6784 my $restore_destroy_volumes = sub {
6785     my ($storecfg, $virtdev_hash) = @_;
6786
6787     for my $dev (values $virtdev_hash->%*) {
6788         my $volid = $dev->{volid} or next;
6789         eval {
6790             PVE::Storage::vdisk_free($storecfg, $volid);
6791             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6792         };
6793         print STDERR "unable to cleanup '$volid' - $@" if $@;
6794     }
6795 };
6796
6797 my $restore_merge_config = sub {
6798     my ($filename, $backup_conf_raw, $override_conf) = @_;
6799
6800     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6801     for my $key (keys $override_conf->%*) {
6802         $backup_conf->{$key} = $override_conf->{$key};
6803     }
6804
6805     return $backup_conf;
6806 };
6807
6808 sub scan_volids {
6809     my ($cfg, $vmid) = @_;
6810
6811     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6812
6813     my $volid_hash = {};
6814     foreach my $storeid (keys %$info) {
6815         foreach my $item (@{$info->{$storeid}}) {
6816             next if !($item->{volid} && $item->{size});
6817             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6818             $volid_hash->{$item->{volid}} = $item;
6819         }
6820     }
6821
6822     return $volid_hash;
6823 }
6824
6825 sub update_disk_config {
6826     my ($vmid, $conf, $volid_hash) = @_;
6827
6828     my $changes;
6829     my $prefix = "VM $vmid";
6830
6831     # used and unused disks
6832     my $referenced = {};
6833
6834     # Note: it is allowed to define multiple storages with same path (alias), so
6835     # we need to check both 'volid' and real 'path' (two different volid can point
6836     # to the same path).
6837
6838     my $referencedpath = {};
6839
6840     # update size info
6841     PVE::QemuConfig->foreach_volume($conf, sub {
6842         my ($opt, $drive) = @_;
6843
6844         my $volid = $drive->{file};
6845         return if !$volid;
6846         my $volume = $volid_hash->{$volid};
6847
6848         # mark volid as "in-use" for next step
6849         $referenced->{$volid} = 1;
6850         if ($volume && (my $path = $volume->{path})) {
6851             $referencedpath->{$path} = 1;
6852         }
6853
6854         return if drive_is_cdrom($drive);
6855         return if !$volume;
6856
6857         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6858         if (defined($updated)) {
6859             $changes = 1;
6860             $conf->{$opt} = print_drive($updated);
6861             print "$prefix ($opt): $msg\n";
6862         }
6863     });
6864
6865     # remove 'unusedX' entry if volume is used
6866     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6867         my ($opt, $drive) = @_;
6868
6869         my $volid = $drive->{file};
6870         return if !$volid;
6871
6872         my $path;
6873         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6874         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6875             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6876             $changes = 1;
6877             delete $conf->{$opt};
6878         }
6879
6880         $referenced->{$volid} = 1;
6881         $referencedpath->{$path} = 1 if $path;
6882     });
6883
6884     foreach my $volid (sort keys %$volid_hash) {
6885         next if $volid =~ m/vm-$vmid-state-/;
6886         next if $referenced->{$volid};
6887         my $path = $volid_hash->{$volid}->{path};
6888         next if !$path; # just to be sure
6889         next if $referencedpath->{$path};
6890         $changes = 1;
6891         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6892         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6893         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6894     }
6895
6896     return $changes;
6897 }
6898
6899 sub rescan {
6900     my ($vmid, $nolock, $dryrun) = @_;
6901
6902     my $cfg = PVE::Storage::config();
6903
6904     print "rescan volumes...\n";
6905     my $volid_hash = scan_volids($cfg, $vmid);
6906
6907     my $updatefn =  sub {
6908         my ($vmid) = @_;
6909
6910         my $conf = PVE::QemuConfig->load_config($vmid);
6911
6912         PVE::QemuConfig->check_lock($conf);
6913
6914         my $vm_volids = {};
6915         foreach my $volid (keys %$volid_hash) {
6916             my $info = $volid_hash->{$volid};
6917             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6918         }
6919
6920         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6921
6922         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6923     };
6924
6925     if (defined($vmid)) {
6926         if ($nolock) {
6927             &$updatefn($vmid);
6928         } else {
6929             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6930         }
6931     } else {
6932         my $vmlist = config_list();
6933         foreach my $vmid (keys %$vmlist) {
6934             if ($nolock) {
6935                 &$updatefn($vmid);
6936             } else {
6937                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6938             }
6939         }
6940     }
6941 }
6942
6943 sub restore_proxmox_backup_archive {
6944     my ($archive, $vmid, $user, $options) = @_;
6945
6946     my $storecfg = PVE::Storage::config();
6947
6948     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6949     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6950
6951     my $fingerprint = $scfg->{fingerprint};
6952     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6953
6954     my $repo = PVE::PBSClient::get_repository($scfg);
6955     my $namespace = $scfg->{namespace};
6956
6957     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6958     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6959     local $ENV{PBS_PASSWORD} = $password;
6960     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6961
6962     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6963         PVE::Storage::parse_volname($storecfg, $archive);
6964
6965     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6966
6967     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6968
6969     my $tmpdir = "/var/tmp/vzdumptmp$$";
6970     rmtree $tmpdir;
6971     mkpath $tmpdir;
6972
6973     my $conffile = PVE::QemuConfig->config_file($vmid);
6974      # disable interrupts (always do cleanups)
6975     local $SIG{INT} =
6976         local $SIG{TERM} =
6977         local $SIG{QUIT} =
6978         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6979
6980     # Note: $oldconf is undef if VM does not exists
6981     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6982     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6983     my $new_conf_raw = '';
6984
6985     my $rpcenv = PVE::RPCEnvironment::get();
6986     my $devinfo = {}; # info about drives included in backup
6987     my $virtdev_hash = {}; # info about allocated drives
6988
6989     eval {
6990         # enable interrupts
6991         local $SIG{INT} =
6992             local $SIG{TERM} =
6993             local $SIG{QUIT} =
6994             local $SIG{HUP} =
6995             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6996
6997         my $cfgfn = "$tmpdir/qemu-server.conf";
6998         my $firewall_config_fn = "$tmpdir/fw.conf";
6999         my $index_fn = "$tmpdir/index.json";
7000
7001         my $cmd = "restore";
7002
7003         my $param = [$pbs_backup_name, "index.json", $index_fn];
7004         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7005         my $index = PVE::Tools::file_get_contents($index_fn);
7006         $index = decode_json($index);
7007
7008         foreach my $info (@{$index->{files}}) {
7009             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7010                 my $devname = $1;
7011                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7012                     $devinfo->{$devname}->{size} = $1;
7013                 } else {
7014                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7015                 }
7016             }
7017         }
7018
7019         my $is_qemu_server_backup = scalar(
7020             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7021         );
7022         if (!$is_qemu_server_backup) {
7023             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7024         }
7025         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7026
7027         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7028         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7029
7030         if ($has_firewall_config) {
7031             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7032             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7033
7034             my $pve_firewall_dir = '/etc/pve/firewall';
7035             mkdir $pve_firewall_dir; # make sure the dir exists
7036             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7037         }
7038
7039         my $fh = IO::File->new($cfgfn, "r") ||
7040             die "unable to read qemu-server.conf - $!\n";
7041
7042         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7043
7044         # fixme: rate limit?
7045
7046         # create empty/temp config
7047         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7048
7049         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7050
7051         # allocate volumes
7052         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7053
7054         foreach my $virtdev (sort keys %$virtdev_hash) {
7055             my $d = $virtdev_hash->{$virtdev};
7056             next if $d->{is_cloudinit}; # no need to restore cloudinit
7057
7058             # this fails if storage is unavailable
7059             my $volid = $d->{volid};
7060             my $path = PVE::Storage::path($storecfg, $volid);
7061
7062             # for live-restore we only want to preload the efidisk and TPM state
7063             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7064
7065             my @ns_arg;
7066             if (defined(my $ns = $scfg->{namespace})) {
7067                 @ns_arg = ('--ns', $ns);
7068             }
7069
7070             my $pbs_restore_cmd = [
7071                 '/usr/bin/pbs-restore',
7072                 '--repository', $repo,
7073                 @ns_arg,
7074                 $pbs_backup_name,
7075                 "$d->{devname}.img.fidx",
7076                 $path,
7077                 '--verbose',
7078                 ];
7079
7080             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7081             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7082
7083             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7084                 push @$pbs_restore_cmd, '--skip-zero';
7085             }
7086
7087             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7088             print "restore proxmox backup image: $dbg_cmdstring\n";
7089             run_command($pbs_restore_cmd);
7090         }
7091
7092         $fh->seek(0, 0) || die "seek failed - $!\n";
7093
7094         my $cookie = { netcount => 0 };
7095         while (defined(my $line = <$fh>)) {
7096             $new_conf_raw .= restore_update_config_line(
7097                 $cookie,
7098                 $map,
7099                 $line,
7100                 $options->{unique},
7101             );
7102         }
7103
7104         $fh->close();
7105     };
7106     my $err = $@;
7107
7108     if ($err || !$options->{live}) {
7109         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7110     }
7111
7112     rmtree $tmpdir;
7113
7114     if ($err) {
7115         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7116         die $err;
7117     }
7118
7119     if ($options->{live}) {
7120         # keep lock during live-restore
7121         $new_conf_raw .= "\nlock: create";
7122     }
7123
7124     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7125     PVE::QemuConfig->write_config($vmid, $new_conf);
7126
7127     eval { rescan($vmid, 1); };
7128     warn $@ if $@;
7129
7130     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7131
7132     if ($options->{live}) {
7133         # enable interrupts
7134         local $SIG{INT} =
7135             local $SIG{TERM} =
7136             local $SIG{QUIT} =
7137             local $SIG{HUP} =
7138             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7139
7140         my $conf = PVE::QemuConfig->load_config($vmid);
7141         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7142
7143         # these special drives are already restored before start
7144         delete $devinfo->{'drive-efidisk0'};
7145         delete $devinfo->{'drive-tpmstate0-backup'};
7146
7147         my $pbs_opts = {
7148             repo => $repo,
7149             keyfile => $keyfile,
7150             snapshot => $pbs_backup_name,
7151             namespace => $namespace,
7152         };
7153         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7154
7155         PVE::QemuConfig->remove_lock($vmid, "create");
7156     }
7157 }
7158
7159 sub pbs_live_restore {
7160     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7161
7162     print "starting VM for live-restore\n";
7163     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7164
7165     my $pbs_backing = {};
7166     for my $ds (keys %$restored_disks) {
7167         $ds =~ m/^drive-(.*)$/;
7168         my $confname = $1;
7169         $pbs_backing->{$confname} = {
7170             repository => $opts->{repo},
7171             snapshot => $opts->{snapshot},
7172             archive => "$ds.img.fidx",
7173         };
7174         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7175         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7176
7177         my $drive = parse_drive($confname, $conf->{$confname});
7178         print "restoring '$ds' to '$drive->{file}'\n";
7179     }
7180
7181     my $drives_streamed = 0;
7182     eval {
7183         # make sure HA doesn't interrupt our restore by stopping the VM
7184         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7185             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7186         }
7187
7188         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7189         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7190         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7191
7192         my $qmeventd_fd = register_qmeventd_handle($vmid);
7193
7194         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7195         # this will effectively collapse the backing image chain consisting of
7196         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7197         # removes itself once all backing images vanish with 'auto-remove=on')
7198         my $jobs = {};
7199         for my $ds (sort keys %$restored_disks) {
7200             my $job_id = "restore-$ds";
7201             mon_cmd($vmid, 'block-stream',
7202                 'job-id' => $job_id,
7203                 device => "$ds",
7204             );
7205             $jobs->{$job_id} = {};
7206         }
7207
7208         mon_cmd($vmid, 'cont');
7209         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7210
7211         print "restore-drive jobs finished successfully, removing all tracking block devices"
7212             ." to disconnect from Proxmox Backup Server\n";
7213
7214         for my $ds (sort keys %$restored_disks) {
7215             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7216         }
7217
7218         close($qmeventd_fd);
7219     };
7220
7221     my $err = $@;
7222
7223     if ($err) {
7224         warn "An error occurred during live-restore: $err\n";
7225         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7226         die "live-restore failed\n";
7227     }
7228 }
7229
7230 sub restore_vma_archive {
7231     my ($archive, $vmid, $user, $opts, $comp) = @_;
7232
7233     my $readfrom = $archive;
7234
7235     my $cfg = PVE::Storage::config();
7236     my $commands = [];
7237     my $bwlimit = $opts->{bwlimit};
7238
7239     my $dbg_cmdstring = '';
7240     my $add_pipe = sub {
7241         my ($cmd) = @_;
7242         push @$commands, $cmd;
7243         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7244         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7245         $readfrom = '-';
7246     };
7247
7248     my $input = undef;
7249     if ($archive eq '-') {
7250         $input = '<&STDIN';
7251     } else {
7252         # If we use a backup from a PVE defined storage we also consider that
7253         # storage's rate limit:
7254         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7255         if (defined($volid)) {
7256             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7257             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7258             if ($readlimit) {
7259                 print STDERR "applying read rate limit: $readlimit\n";
7260                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7261                 $add_pipe->($cstream);
7262             }
7263         }
7264     }
7265
7266     if ($comp) {
7267         my $info = PVE::Storage::decompressor_info('vma', $comp);
7268         my $cmd = $info->{decompressor};
7269         push @$cmd, $readfrom;
7270         $add_pipe->($cmd);
7271     }
7272
7273     my $tmpdir = "/var/tmp/vzdumptmp$$";
7274     rmtree $tmpdir;
7275
7276     # disable interrupts (always do cleanups)
7277     local $SIG{INT} =
7278         local $SIG{TERM} =
7279         local $SIG{QUIT} =
7280         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7281
7282     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7283     POSIX::mkfifo($mapfifo, 0600);
7284     my $fifofh;
7285     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7286
7287     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7288
7289     my $oldtimeout;
7290     my $timeout = 5;
7291
7292     my $devinfo = {}; # info about drives included in backup
7293     my $virtdev_hash = {}; # info about allocated drives
7294
7295     my $rpcenv = PVE::RPCEnvironment::get();
7296
7297     my $conffile = PVE::QemuConfig->config_file($vmid);
7298
7299     # Note: $oldconf is undef if VM does not exist
7300     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7301     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7302     my $new_conf_raw = '';
7303
7304     my %storage_limits;
7305
7306     my $print_devmap = sub {
7307         my $cfgfn = "$tmpdir/qemu-server.conf";
7308
7309         # we can read the config - that is already extracted
7310         my $fh = IO::File->new($cfgfn, "r") ||
7311             die "unable to read qemu-server.conf - $!\n";
7312
7313         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7314         if (-f $fwcfgfn) {
7315             my $pve_firewall_dir = '/etc/pve/firewall';
7316             mkdir $pve_firewall_dir; # make sure the dir exists
7317             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7318         }
7319
7320         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7321
7322         foreach my $info (values %{$virtdev_hash}) {
7323             my $storeid = $info->{storeid};
7324             next if defined($storage_limits{$storeid});
7325
7326             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7327             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7328             $storage_limits{$storeid} = $limit * 1024;
7329         }
7330
7331         foreach my $devname (keys %$devinfo) {
7332             die "found no device mapping information for device '$devname'\n"
7333                 if !$devinfo->{$devname}->{virtdev};
7334         }
7335
7336         # create empty/temp config
7337         if ($oldconf) {
7338             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7339             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7340         }
7341
7342         # allocate volumes
7343         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7344
7345         # print restore information to $fifofh
7346         foreach my $virtdev (sort keys %$virtdev_hash) {
7347             my $d = $virtdev_hash->{$virtdev};
7348             next if $d->{is_cloudinit}; # no need to restore cloudinit
7349
7350             my $storeid = $d->{storeid};
7351             my $volid = $d->{volid};
7352
7353             my $map_opts = '';
7354             if (my $limit = $storage_limits{$storeid}) {
7355                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7356             }
7357
7358             my $write_zeros = 1;
7359             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7360                 $write_zeros = 0;
7361             }
7362
7363             my $path = PVE::Storage::path($cfg, $volid);
7364
7365             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7366
7367             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7368         }
7369
7370         $fh->seek(0, 0) || die "seek failed - $!\n";
7371
7372         my $cookie = { netcount => 0 };
7373         while (defined(my $line = <$fh>)) {
7374             $new_conf_raw .= restore_update_config_line(
7375                 $cookie,
7376                 $map,
7377                 $line,
7378                 $opts->{unique},
7379             );
7380         }
7381
7382         $fh->close();
7383     };
7384
7385     eval {
7386         # enable interrupts
7387         local $SIG{INT} =
7388             local $SIG{TERM} =
7389             local $SIG{QUIT} =
7390             local $SIG{HUP} =
7391             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7392         local $SIG{ALRM} = sub { die "got timeout\n"; };
7393
7394         $oldtimeout = alarm($timeout);
7395
7396         my $parser = sub {
7397             my $line = shift;
7398
7399             print "$line\n";
7400
7401             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7402                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7403                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7404             } elsif ($line =~ m/^CTIME: /) {
7405                 # we correctly received the vma config, so we can disable
7406                 # the timeout now for disk allocation (set to 10 minutes, so
7407                 # that we always timeout if something goes wrong)
7408                 alarm(600);
7409                 &$print_devmap();
7410                 print $fifofh "done\n";
7411                 my $tmp = $oldtimeout || 0;
7412                 $oldtimeout = undef;
7413                 alarm($tmp);
7414                 close($fifofh);
7415                 $fifofh = undef;
7416             }
7417         };
7418
7419         print "restore vma archive: $dbg_cmdstring\n";
7420         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7421     };
7422     my $err = $@;
7423
7424     alarm($oldtimeout) if $oldtimeout;
7425
7426     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7427
7428     close($fifofh) if $fifofh;
7429     unlink $mapfifo;
7430     rmtree $tmpdir;
7431
7432     if ($err) {
7433         $restore_destroy_volumes->($cfg, $virtdev_hash);
7434         die $err;
7435     }
7436
7437     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7438     PVE::QemuConfig->write_config($vmid, $new_conf);
7439
7440     eval { rescan($vmid, 1); };
7441     warn $@ if $@;
7442
7443     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7444 }
7445
7446 sub restore_tar_archive {
7447     my ($archive, $vmid, $user, $opts) = @_;
7448
7449     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7450         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7451         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7452     }
7453
7454     if ($archive ne '-') {
7455         my $firstfile = tar_archive_read_firstfile($archive);
7456         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7457             if $firstfile ne 'qemu-server.conf';
7458     }
7459
7460     my $storecfg = PVE::Storage::config();
7461
7462     # avoid zombie disks when restoring over an existing VM -> cleanup first
7463     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7464     # skiplock=1 because qmrestore has set the 'create' lock itself already
7465     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7466     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7467
7468     my $tocmd = "/usr/lib/qemu-server/qmextract";
7469
7470     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7471     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7472     $tocmd .= ' --prealloc' if $opts->{prealloc};
7473     $tocmd .= ' --info' if $opts->{info};
7474
7475     # tar option "xf" does not autodetect compression when read from STDIN,
7476     # so we pipe to zcat
7477     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7478         PVE::Tools::shellquote("--to-command=$tocmd");
7479
7480     my $tmpdir = "/var/tmp/vzdumptmp$$";
7481     mkpath $tmpdir;
7482
7483     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7484     local $ENV{VZDUMP_VMID} = $vmid;
7485     local $ENV{VZDUMP_USER} = $user;
7486
7487     my $conffile = PVE::QemuConfig->config_file($vmid);
7488     my $new_conf_raw = '';
7489
7490     # disable interrupts (always do cleanups)
7491     local $SIG{INT} =
7492         local $SIG{TERM} =
7493         local $SIG{QUIT} =
7494         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7495
7496     eval {
7497         # enable interrupts
7498         local $SIG{INT} =
7499             local $SIG{TERM} =
7500             local $SIG{QUIT} =
7501             local $SIG{HUP} =
7502             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7503
7504         if ($archive eq '-') {
7505             print "extracting archive from STDIN\n";
7506             run_command($cmd, input => "<&STDIN");
7507         } else {
7508             print "extracting archive '$archive'\n";
7509             run_command($cmd);
7510         }
7511
7512         return if $opts->{info};
7513
7514         # read new mapping
7515         my $map = {};
7516         my $statfile = "$tmpdir/qmrestore.stat";
7517         if (my $fd = IO::File->new($statfile, "r")) {
7518             while (defined (my $line = <$fd>)) {
7519                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7520                     $map->{$1} = $2 if $1;
7521                 } else {
7522                     print STDERR "unable to parse line in statfile - $line\n";
7523                 }
7524             }
7525             $fd->close();
7526         }
7527
7528         my $confsrc = "$tmpdir/qemu-server.conf";
7529
7530         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7531
7532         my $cookie = { netcount => 0 };
7533         while (defined (my $line = <$srcfd>)) {
7534             $new_conf_raw .= restore_update_config_line(
7535                 $cookie,
7536                 $map,
7537                 $line,
7538                 $opts->{unique},
7539             );
7540         }
7541
7542         $srcfd->close();
7543     };
7544     if (my $err = $@) {
7545         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7546         die $err;
7547     }
7548
7549     rmtree $tmpdir;
7550
7551     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7552
7553     PVE::Cluster::cfs_update(); # make sure we read new file
7554
7555     eval { rescan($vmid, 1); };
7556     warn $@ if $@;
7557 };
7558
7559 sub foreach_storage_used_by_vm {
7560     my ($conf, $func) = @_;
7561
7562     my $sidhash = {};
7563
7564     PVE::QemuConfig->foreach_volume($conf, sub {
7565         my ($ds, $drive) = @_;
7566         return if drive_is_cdrom($drive);
7567
7568         my $volid = $drive->{file};
7569
7570         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7571         $sidhash->{$sid} = $sid if $sid;
7572     });
7573
7574     foreach my $sid (sort keys %$sidhash) {
7575         &$func($sid);
7576     }
7577 }
7578
7579 my $qemu_snap_storage = {
7580     rbd => 1,
7581 };
7582 sub do_snapshots_with_qemu {
7583     my ($storecfg, $volid, $deviceid) = @_;
7584
7585     return if $deviceid =~ m/tpmstate0/;
7586
7587     my $storage_name = PVE::Storage::parse_volume_id($volid);
7588     my $scfg = $storecfg->{ids}->{$storage_name};
7589     die "could not find storage '$storage_name'\n" if !defined($scfg);
7590
7591     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7592         return 1;
7593     }
7594
7595     if ($volid =~ m/\.(qcow2|qed)$/){
7596         return 1;
7597     }
7598
7599     return;
7600 }
7601
7602 sub qga_check_running {
7603     my ($vmid, $nowarn) = @_;
7604
7605     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7606     if ($@) {
7607         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7608         return 0;
7609     }
7610     return 1;
7611 }
7612
7613 sub template_create {
7614     my ($vmid, $conf, $disk) = @_;
7615
7616     my $storecfg = PVE::Storage::config();
7617
7618     PVE::QemuConfig->foreach_volume($conf, sub {
7619         my ($ds, $drive) = @_;
7620
7621         return if drive_is_cdrom($drive);
7622         return if $disk && $ds ne $disk;
7623
7624         my $volid = $drive->{file};
7625         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7626
7627         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7628         $drive->{file} = $voliddst;
7629         $conf->{$ds} = print_drive($drive);
7630         PVE::QemuConfig->write_config($vmid, $conf);
7631     });
7632 }
7633
7634 sub convert_iscsi_path {
7635     my ($path) = @_;
7636
7637     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7638         my $portal = $1;
7639         my $target = $2;
7640         my $lun = $3;
7641
7642         my $initiator_name = get_initiator_name();
7643
7644         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7645                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7646     }
7647
7648     die "cannot convert iscsi path '$path', unkown format\n";
7649 }
7650
7651 sub qemu_img_convert {
7652     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7653
7654     my $storecfg = PVE::Storage::config();
7655     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7656     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7657
7658     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7659
7660     my $cachemode;
7661     my $src_path;
7662     my $src_is_iscsi = 0;
7663     my $src_format;
7664
7665     if ($src_storeid) {
7666         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7667         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7668         $src_format = qemu_img_format($src_scfg, $src_volname);
7669         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7670         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7671         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7672     } elsif (-f $src_volid || -b $src_volid) {
7673         $src_path = $src_volid;
7674         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7675             $src_format = $1;
7676         }
7677     }
7678
7679     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7680
7681     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7682     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7683     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7684     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7685
7686     my $cmd = [];
7687     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7688     push @$cmd, '-l', "snapshot.name=$snapname"
7689         if $snapname && $src_format && $src_format eq "qcow2";
7690     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7691     push @$cmd, '-T', $cachemode if defined($cachemode);
7692
7693     if ($src_is_iscsi) {
7694         push @$cmd, '--image-opts';
7695         $src_path = convert_iscsi_path($src_path);
7696     } elsif ($src_format) {
7697         push @$cmd, '-f', $src_format;
7698     }
7699
7700     if ($dst_is_iscsi) {
7701         push @$cmd, '--target-image-opts';
7702         $dst_path = convert_iscsi_path($dst_path);
7703     } else {
7704         push @$cmd, '-O', $dst_format;
7705     }
7706
7707     push @$cmd, $src_path;
7708
7709     if (!$dst_is_iscsi && $is_zero_initialized) {
7710         push @$cmd, "zeroinit:$dst_path";
7711     } else {
7712         push @$cmd, $dst_path;
7713     }
7714
7715     my $parser = sub {
7716         my $line = shift;
7717         if($line =~ m/\((\S+)\/100\%\)/){
7718             my $percent = $1;
7719             my $transferred = int($size * $percent / 100);
7720             my $total_h = render_bytes($size, 1);
7721             my $transferred_h = render_bytes($transferred, 1);
7722
7723             print "transferred $transferred_h of $total_h ($percent%)\n";
7724         }
7725
7726     };
7727
7728     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7729     my $err = $@;
7730     die "copy failed: $err" if $err;
7731 }
7732
7733 sub qemu_img_format {
7734     my ($scfg, $volname) = @_;
7735
7736     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7737         return $1;
7738     } else {
7739         return "raw";
7740     }
7741 }
7742
7743 sub qemu_drive_mirror {
7744     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7745
7746     $jobs = {} if !$jobs;
7747
7748     my $qemu_target;
7749     my $format;
7750     $jobs->{"drive-$drive"} = {};
7751
7752     if ($dst_volid =~ /^nbd:/) {
7753         $qemu_target = $dst_volid;
7754         $format = "nbd";
7755     } else {
7756         my $storecfg = PVE::Storage::config();
7757         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7758
7759         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7760
7761         $format = qemu_img_format($dst_scfg, $dst_volname);
7762
7763         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7764
7765         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7766     }
7767
7768     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7769     $opts->{format} = $format if $format;
7770
7771     if (defined($src_bitmap)) {
7772         $opts->{sync} = 'incremental';
7773         $opts->{bitmap} = $src_bitmap;
7774         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7775     }
7776
7777     if (defined($bwlimit)) {
7778         $opts->{speed} = $bwlimit * 1024;
7779         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7780     } else {
7781         print "drive mirror is starting for drive-$drive\n";
7782     }
7783
7784     # if a job already runs for this device we get an error, catch it for cleanup
7785     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7786     if (my $err = $@) {
7787         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7788         warn "$@\n" if $@;
7789         die "mirroring error: $err\n";
7790     }
7791
7792     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7793 }
7794
7795 # $completion can be either
7796 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7797 # 'cancel': wait until all jobs are ready, block-job-cancel them
7798 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7799 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7800 sub qemu_drive_mirror_monitor {
7801     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7802
7803     $completion //= 'complete';
7804     $op //= "mirror";
7805
7806     eval {
7807         my $err_complete = 0;
7808
7809         my $starttime = time ();
7810         while (1) {
7811             die "block job ('$op') timed out\n" if $err_complete > 300;
7812
7813             my $stats = mon_cmd($vmid, "query-block-jobs");
7814             my $ctime = time();
7815
7816             my $running_jobs = {};
7817             for my $stat (@$stats) {
7818                 next if $stat->{type} ne $op;
7819                 $running_jobs->{$stat->{device}} = $stat;
7820             }
7821
7822             my $readycounter = 0;
7823
7824             for my $job_id (sort keys %$jobs) {
7825                 my $job = $running_jobs->{$job_id};
7826
7827                 my $vanished = !defined($job);
7828                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7829                 if($complete || ($vanished && $completion eq 'auto')) {
7830                     print "$job_id: $op-job finished\n";
7831                     delete $jobs->{$job_id};
7832                     next;
7833                 }
7834
7835                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7836
7837                 my $busy = $job->{busy};
7838                 my $ready = $job->{ready};
7839                 if (my $total = $job->{len}) {
7840                     my $transferred = $job->{offset} || 0;
7841                     my $remaining = $total - $transferred;
7842                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7843
7844                     my $duration = $ctime - $starttime;
7845                     my $total_h = render_bytes($total, 1);
7846                     my $transferred_h = render_bytes($transferred, 1);
7847
7848                     my $status = sprintf(
7849                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7850                         render_duration($duration),
7851                     );
7852
7853                     if ($ready) {
7854                         if ($busy) {
7855                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7856                         } else {
7857                             $status .= ", ready";
7858                         }
7859                     }
7860                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7861                     $jobs->{$job_id}->{ready} = $ready;
7862                 }
7863
7864                 $readycounter++ if $job->{ready};
7865             }
7866
7867             last if scalar(keys %$jobs) == 0;
7868
7869             if ($readycounter == scalar(keys %$jobs)) {
7870                 print "all '$op' jobs are ready\n";
7871
7872                 # do the complete later (or has already been done)
7873                 last if $completion eq 'skip' || $completion eq 'auto';
7874
7875                 if ($vmiddst && $vmiddst != $vmid) {
7876                     my $agent_running = $qga && qga_check_running($vmid);
7877                     if ($agent_running) {
7878                         print "freeze filesystem\n";
7879                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7880                         warn $@ if $@;
7881                     } else {
7882                         print "suspend vm\n";
7883                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7884                         warn $@ if $@;
7885                     }
7886
7887                     # if we clone a disk for a new target vm, we don't switch the disk
7888                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7889
7890                     if ($agent_running) {
7891                         print "unfreeze filesystem\n";
7892                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7893                         warn $@ if $@;
7894                     } else {
7895                         print "resume vm\n";
7896                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7897                         warn $@ if $@;
7898                     }
7899
7900                     last;
7901                 } else {
7902
7903                     for my $job_id (sort keys %$jobs) {
7904                         # try to switch the disk if source and destination are on the same guest
7905                         print "$job_id: Completing block job_id...\n";
7906
7907                         my $op;
7908                         if ($completion eq 'complete') {
7909                             $op = 'block-job-complete';
7910                         } elsif ($completion eq 'cancel') {
7911                             $op = 'block-job-cancel';
7912                         } else {
7913                             die "invalid completion value: $completion\n";
7914                         }
7915                         eval { mon_cmd($vmid, $op, device => $job_id) };
7916                         if ($@ =~ m/cannot be completed/) {
7917                             print "$job_id: block job cannot be completed, trying again.\n";
7918                             $err_complete++;
7919                         }else {
7920                             print "$job_id: Completed successfully.\n";
7921                             $jobs->{$job_id}->{complete} = 1;
7922                         }
7923                     }
7924                 }
7925             }
7926             sleep 1;
7927         }
7928     };
7929     my $err = $@;
7930
7931     if ($err) {
7932         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7933         die "block job ($op) error: $err";
7934     }
7935 }
7936
7937 sub qemu_blockjobs_cancel {
7938     my ($vmid, $jobs) = @_;
7939
7940     foreach my $job (keys %$jobs) {
7941         print "$job: Cancelling block job\n";
7942         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7943         $jobs->{$job}->{cancel} = 1;
7944     }
7945
7946     while (1) {
7947         my $stats = mon_cmd($vmid, "query-block-jobs");
7948
7949         my $running_jobs = {};
7950         foreach my $stat (@$stats) {
7951             $running_jobs->{$stat->{device}} = $stat;
7952         }
7953
7954         foreach my $job (keys %$jobs) {
7955
7956             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7957                 print "$job: Done.\n";
7958                 delete $jobs->{$job};
7959             }
7960         }
7961
7962         last if scalar(keys %$jobs) == 0;
7963
7964         sleep 1;
7965     }
7966 }
7967
7968 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
7969 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
7970 my sub clone_disk_check_io_uring {
7971     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
7972
7973     return if !$use_drive_mirror;
7974
7975     # Don't complain when not changing storage.
7976     # Assume if it works for the source, it'll work for the target too.
7977     return if $src_storeid eq $dst_storeid;
7978
7979     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7980     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7981
7982     my $cache_direct = drive_uses_cache_direct($src_drive);
7983
7984     my $src_uses_io_uring;
7985     if ($src_drive->{aio}) {
7986         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
7987     } else {
7988         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
7989     }
7990
7991     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
7992         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
7993 }
7994
7995 sub clone_disk {
7996     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7997
7998     my ($vmid, $running) = $source->@{qw(vmid running)};
7999     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8000
8001     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8002     my ($storage, $format) = $dest->@{qw(storage format)};
8003
8004     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8005
8006     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8007         die "cloning from/to EFI disk requires EFI disk\n"
8008             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8009         die "cloning from/to TPM state requires TPM state\n"
8010             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8011
8012         # This would lead to two device nodes in QEMU pointing to the same backing image!
8013         die "cannot change drive name when cloning disk from/to the same VM\n"
8014             if $use_drive_mirror && $vmid == $newvmid;
8015     }
8016
8017     die "cannot move TPM state while VM is running\n"
8018         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8019
8020     my $newvolid;
8021
8022     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8023     print "$src_drivename " if $src_drivename;
8024     print "($drive->{file})\n";
8025
8026     if (!$full) {
8027         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8028         push @$newvollist, $newvolid;
8029     } else {
8030         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8031         my $storeid = $storage || $src_storeid;
8032
8033         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8034
8035         my $name = undef;
8036         my $size = undef;
8037         if (drive_is_cloudinit($drive)) {
8038             $name = "vm-$newvmid-cloudinit";
8039             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8040             if ($scfg->{path}) {
8041                 $name .= ".$dst_format";
8042             }
8043             $snapname = undef;
8044             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8045         } elsif ($dst_drivename eq 'efidisk0') {
8046             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8047         } elsif ($dst_drivename eq 'tpmstate0') {
8048             $dst_format = 'raw';
8049             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8050         } else {
8051             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8052
8053             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8054         }
8055         $newvolid = PVE::Storage::vdisk_alloc(
8056             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8057         );
8058         push @$newvollist, $newvolid;
8059
8060         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8061
8062         if (drive_is_cloudinit($drive)) {
8063             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8064             # if this is the case, we have to complete any block-jobs still there from
8065             # previous drive-mirrors
8066             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8067                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8068             }
8069             goto no_data_clone;
8070         }
8071
8072         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8073         if ($use_drive_mirror) {
8074             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8075                 $completion, $qga, $bwlimit);
8076         } else {
8077             # TODO: handle bwlimits
8078             if ($dst_drivename eq 'efidisk0') {
8079                 # the relevant data on the efidisk may be smaller than the source
8080                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8081                 # that is given by the OVMF_VARS.fd
8082                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8083                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8084
8085                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8086
8087                 # better for Ceph if block size is not too small, see bug #3324
8088                 my $bs = 1024*1024;
8089
8090                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8091
8092                 if ($src_format eq 'qcow2' && $snapname) {
8093                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8094                         if !min_version(kvm_user_version(), 6, 2);
8095                     push $cmd->@*, '-l', $snapname;
8096                 }
8097                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8098                 run_command($cmd);
8099             } else {
8100                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
8101             }
8102         }
8103     }
8104
8105 no_data_clone:
8106     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8107
8108     my $disk = dclone($drive);
8109     delete $disk->{format};
8110     $disk->{file} = $newvolid;
8111     $disk->{size} = $size if defined($size);
8112
8113     return $disk;
8114 }
8115
8116 sub get_running_qemu_version {
8117     my ($vmid) = @_;
8118     my $res = mon_cmd($vmid, "query-version");
8119     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8120 }
8121
8122 sub qemu_use_old_bios_files {
8123     my ($machine_type) = @_;
8124
8125     return if !$machine_type;
8126
8127     my $use_old_bios_files = undef;
8128
8129     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8130         $machine_type = $1;
8131         $use_old_bios_files = 1;
8132     } else {
8133         my $version = extract_version($machine_type, kvm_user_version());
8134         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8135         # load new efi bios files on migration. So this hack is required to allow
8136         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8137         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8138         $use_old_bios_files = !min_version($version, 2, 4);
8139     }
8140
8141     return ($use_old_bios_files, $machine_type);
8142 }
8143
8144 sub get_efivars_size {
8145     my ($conf, $efidisk) = @_;
8146
8147     my $arch = get_vm_arch($conf);
8148     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8149     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8150     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8151     return -s $ovmf_vars;
8152 }
8153
8154 sub update_efidisk_size {
8155     my ($conf) = @_;
8156
8157     return if !defined($conf->{efidisk0});
8158
8159     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8160     $disk->{size} = get_efivars_size($conf);
8161     $conf->{efidisk0} = print_drive($disk);
8162
8163     return;
8164 }
8165
8166 sub update_tpmstate_size {
8167     my ($conf) = @_;
8168
8169     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8170     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8171     $conf->{tpmstate0} = print_drive($disk);
8172 }
8173
8174 sub create_efidisk($$$$$$$) {
8175     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8176
8177     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8178
8179     my $vars_size_b = -s $ovmf_vars;
8180     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8181     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8182     PVE::Storage::activate_volumes($storecfg, [$volid]);
8183
8184     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8185     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8186
8187     return ($volid, $size/1024);
8188 }
8189
8190 sub vm_iothreads_list {
8191     my ($vmid) = @_;
8192
8193     my $res = mon_cmd($vmid, 'query-iothreads');
8194
8195     my $iothreads = {};
8196     foreach my $iothread (@$res) {
8197         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8198     }
8199
8200     return $iothreads;
8201 }
8202
8203 sub scsihw_infos {
8204     my ($conf, $drive) = @_;
8205
8206     my $maxdev = 0;
8207
8208     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8209         $maxdev = 7;
8210     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8211         $maxdev = 1;
8212     } else {
8213         $maxdev = 256;
8214     }
8215
8216     my $controller = int($drive->{index} / $maxdev);
8217     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8218         ? "virtioscsi"
8219         : "scsihw";
8220
8221     return ($maxdev, $controller, $controller_prefix);
8222 }
8223
8224 sub resolve_dst_disk_format {
8225         my ($storecfg, $storeid, $src_volname, $format) = @_;
8226         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8227
8228         if (!$format) {
8229             # if no target format is specified, use the source disk format as hint
8230             if ($src_volname) {
8231                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8232                 $format = qemu_img_format($scfg, $src_volname);
8233             } else {
8234                 return $defFormat;
8235             }
8236         }
8237
8238         # test if requested format is supported - else use default
8239         my $supported = grep { $_ eq $format } @$validFormats;
8240         $format = $defFormat if !$supported;
8241         return $format;
8242 }
8243
8244 # NOTE: if this logic changes, please update docs & possibly gui logic
8245 sub find_vmstate_storage {
8246     my ($conf, $storecfg) = @_;
8247
8248     # first, return storage from conf if set
8249     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8250
8251     my ($target, $shared, $local);
8252
8253     foreach_storage_used_by_vm($conf, sub {
8254         my ($sid) = @_;
8255         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8256         my $dst = $scfg->{shared} ? \$shared : \$local;
8257         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8258     });
8259
8260     # second, use shared storage where VM has at least one disk
8261     # third, use local storage where VM has at least one disk
8262     # fall back to local storage
8263     $target = $shared // $local // 'local';
8264
8265     return $target;
8266 }
8267
8268 sub generate_uuid {
8269     my ($uuid, $uuid_str);
8270     UUID::generate($uuid);
8271     UUID::unparse($uuid, $uuid_str);
8272     return $uuid_str;
8273 }
8274
8275 sub generate_smbios1_uuid {
8276     return "uuid=".generate_uuid();
8277 }
8278
8279 sub nbd_stop {
8280     my ($vmid) = @_;
8281
8282     mon_cmd($vmid, 'nbd-server-stop');
8283 }
8284
8285 sub create_reboot_request {
8286     my ($vmid) = @_;
8287     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8288         or die "failed to create reboot trigger file: $!\n";
8289     close($fh);
8290 }
8291
8292 sub clear_reboot_request {
8293     my ($vmid) = @_;
8294     my $path = "/run/qemu-server/$vmid.reboot";
8295     my $res = 0;
8296
8297     $res = unlink($path);
8298     die "could not remove reboot request for $vmid: $!"
8299         if !$res && $! != POSIX::ENOENT;
8300
8301     return $res;
8302 }
8303
8304 sub bootorder_from_legacy {
8305     my ($conf, $bootcfg) = @_;
8306
8307     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8308     my $bootindex_hash = {};
8309     my $i = 1;
8310     foreach my $o (split(//, $boot)) {
8311         $bootindex_hash->{$o} = $i*100;
8312         $i++;
8313     }
8314
8315     my $bootorder = {};
8316
8317     PVE::QemuConfig->foreach_volume($conf, sub {
8318         my ($ds, $drive) = @_;
8319
8320         if (drive_is_cdrom ($drive, 1)) {
8321             if ($bootindex_hash->{d}) {
8322                 $bootorder->{$ds} = $bootindex_hash->{d};
8323                 $bootindex_hash->{d} += 1;
8324             }
8325         } elsif ($bootindex_hash->{c}) {
8326             $bootorder->{$ds} = $bootindex_hash->{c}
8327                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8328             $bootindex_hash->{c} += 1;
8329         }
8330     });
8331
8332     if ($bootindex_hash->{n}) {
8333         for (my $i = 0; $i < $MAX_NETS; $i++) {
8334             my $netname = "net$i";
8335             next if !$conf->{$netname};
8336             $bootorder->{$netname} = $bootindex_hash->{n};
8337             $bootindex_hash->{n} += 1;
8338         }
8339     }
8340
8341     return $bootorder;
8342 }
8343
8344 # Generate default device list for 'boot: order=' property. Matches legacy
8345 # default boot order, but with explicit device names. This is important, since
8346 # the fallback for when neither 'order' nor the old format is specified relies
8347 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8348 sub get_default_bootdevices {
8349     my ($conf) = @_;
8350
8351     my @ret = ();
8352
8353     # harddisk
8354     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8355     push @ret, $first if $first;
8356
8357     # cdrom
8358     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8359     push @ret, $first if $first;
8360
8361     # network
8362     for (my $i = 0; $i < $MAX_NETS; $i++) {
8363         my $netname = "net$i";
8364         next if !$conf->{$netname};
8365         push @ret, $netname;
8366         last;
8367     }
8368
8369     return \@ret;
8370 }
8371
8372 sub device_bootorder {
8373     my ($conf) = @_;
8374
8375     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8376
8377     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8378
8379     my $bootorder = {};
8380     if (!defined($boot) || $boot->{legacy}) {
8381         $bootorder = bootorder_from_legacy($conf, $boot);
8382     } elsif ($boot->{order}) {
8383         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8384         for my $dev (PVE::Tools::split_list($boot->{order})) {
8385             $bootorder->{$dev} = $i++;
8386         }
8387     }
8388
8389     return $bootorder;
8390 }
8391
8392 sub register_qmeventd_handle {
8393     my ($vmid) = @_;
8394
8395     my $fh;
8396     my $peer = "/var/run/qmeventd.sock";
8397     my $count = 0;
8398
8399     for (;;) {
8400         $count++;
8401         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8402         last if $fh;
8403         if ($! != EINTR && $! != EAGAIN) {
8404             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8405         }
8406         if ($count > 4) {
8407             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8408               . "after $count retries\n";
8409         }
8410         usleep(25000);
8411     }
8412
8413     # send handshake to mark VM as backing up
8414     print $fh to_json({vzdump => {vmid => "$vmid"}});
8415
8416     # return handle to be closed later when inhibit is no longer required
8417     return $fh;
8418 }
8419
8420 # bash completion helper
8421
8422 sub complete_backup_archives {
8423     my ($cmdname, $pname, $cvalue) = @_;
8424
8425     my $cfg = PVE::Storage::config();
8426
8427     my $storeid;
8428
8429     if ($cvalue =~ m/^([^:]+):/) {
8430         $storeid = $1;
8431     }
8432
8433     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8434
8435     my $res = [];
8436     foreach my $id (keys %$data) {
8437         foreach my $item (@{$data->{$id}}) {
8438             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8439             push @$res, $item->{volid} if defined($item->{volid});
8440         }
8441     }
8442
8443     return $res;
8444 }
8445
8446 my $complete_vmid_full = sub {
8447     my ($running) = @_;
8448
8449     my $idlist = vmstatus();
8450
8451     my $res = [];
8452
8453     foreach my $id (keys %$idlist) {
8454         my $d = $idlist->{$id};
8455         if (defined($running)) {
8456             next if $d->{template};
8457             next if $running && $d->{status} ne 'running';
8458             next if !$running && $d->{status} eq 'running';
8459         }
8460         push @$res, $id;
8461
8462     }
8463     return $res;
8464 };
8465
8466 sub complete_vmid {
8467     return &$complete_vmid_full();
8468 }
8469
8470 sub complete_vmid_stopped {
8471     return &$complete_vmid_full(0);
8472 }
8473
8474 sub complete_vmid_running {
8475     return &$complete_vmid_full(1);
8476 }
8477
8478 sub complete_storage {
8479
8480     my $cfg = PVE::Storage::config();
8481     my $ids = $cfg->{ids};
8482
8483     my $res = [];
8484     foreach my $sid (keys %$ids) {
8485         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8486         next if !$ids->{$sid}->{content}->{images};
8487         push @$res, $sid;
8488     }
8489
8490     return $res;
8491 }
8492
8493 sub complete_migration_storage {
8494     my ($cmd, $param, $current_value, $all_args) = @_;
8495
8496     my $targetnode = @$all_args[1];
8497
8498     my $cfg = PVE::Storage::config();
8499     my $ids = $cfg->{ids};
8500
8501     my $res = [];
8502     foreach my $sid (keys %$ids) {
8503         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8504         next if !$ids->{$sid}->{content}->{images};
8505         push @$res, $sid;
8506     }
8507
8508     return $res;
8509 }
8510
8511 sub vm_is_paused {
8512     my ($vmid) = @_;
8513     my $qmpstatus = eval {
8514         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8515         mon_cmd($vmid, "query-status");
8516     };
8517     warn "$@\n" if $@;
8518     return $qmpstatus && $qmpstatus->{status} eq "paused";
8519 }
8520
8521 sub check_volume_storage_type {
8522     my ($storecfg, $vol) = @_;
8523
8524     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8525     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8526     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8527
8528     die "storage '$storeid' does not support content-type '$vtype'\n"
8529         if !$scfg->{content}->{$vtype};
8530
8531     return 1;
8532 }
8533
8534 sub add_nets_bridge_fdb {
8535     my ($conf, $vmid) = @_;
8536
8537     for my $opt (keys %$conf) {
8538         next if $opt !~ m/^net(\d+)$/;
8539         my $iface = "tap${vmid}i$1";
8540         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8541         my $net = parse_net($conf->{$opt}, 1) or next;
8542
8543         my $mac = $net->{macaddr};
8544         if (!$mac) {
8545             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8546                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8547             next;
8548         }
8549
8550         my $bridge = $net->{bridge};
8551         if ($have_sdn) {
8552             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8553         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8554             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8555         }
8556     }
8557 }
8558
8559 sub del_nets_bridge_fdb {
8560     my ($conf, $vmid) = @_;
8561
8562     for my $opt (keys %$conf) {
8563         next if $opt !~ m/^net(\d+)$/;
8564         my $iface = "tap${vmid}i$1";
8565
8566         my $net = parse_net($conf->{$opt}) or next;
8567         my $mac = $net->{macaddr} or next;
8568
8569         my $bridge = $net->{bridge};
8570         if ($have_sdn) {
8571             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8572         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8573             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8574         }
8575     }
8576 }
8577
8578 1;