PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::INotify;
  38 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  39 use PVE::ProcFSTools;
  40 use PVE::PBSClient;
  41 use PVE::RESTEnvironment qw(log_warn);
  42 use PVE::RPCEnvironment;
  43 use PVE::Storage;
  44 use PVE::SysFSTools;
  45 use PVE::Systemd;
  46 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  47
  48 use PVE::QMPClient;
  49 use PVE::QemuConfig;
  50 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  51 use PVE::QemuServer::Cloudinit;
  52 use PVE::QemuServer::CGroup;
  53 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  54 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  55 use PVE::QemuServer::Machine;
  56 use PVE::QemuServer::Memory;
  57 use PVE::QemuServer::Monitor qw(mon_cmd);
  58 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  59 use PVE::QemuServer::USB qw(parse_usb_device);
  60
  61 my $have_sdn;
  62 eval {
  63     require PVE::Network::SDN::Zones;
  64     $have_sdn = 1;
  65 };
  66
  67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  68 my $OVMF = {
  69     x86_64 => {
  70         '4m-no-smm' => [
  71             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  72             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  73         ],
  74         '4m-no-smm-ms' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  77         ],
  78         '4m' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  81         ],
  82         '4m-ms' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  85         ],
  86         default => [
  87             "$EDK2_FW_BASE/OVMF_CODE.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS.fd",
  89         ],
  90     },
  91     aarch64 => {
  92         default => [
  93             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  94             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  95         ],
  96     },
  97 };
  98
  99 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 100
 101 # Note about locking: we use flock on the config file protect against concurent actions.
 102 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 104 # But you can ignore this kind of lock with the --skiplock flag.
 105
 106 cfs_register_file(
 107     '/qemu-server/',
 108     \&parse_vm_config,
 109     \&write_vm_config
 110 );
 111
 112 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 113     description => "Some command save/restore state from this location.",
 114     type => 'string',
 115     maxLength => 128,
 116     optional => 1,
 117 });
 118
 119 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 120         description => "Specifies the QEMU machine type.",
 121         type => 'string',
 122         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 123         maxLength => 40,
 124         optional => 1,
 125 });
 126
 127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 244             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 245             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 246             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 247             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 248             ." a hardware RNG from the host.",
 249     },
 250     max_bytes => {
 251         type => 'integer',
 252         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 253             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 254             ." `0` to disable limiting (potentially dangerous!).",
 255         optional => 1,
 256
 257         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 258         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 259         # reading from /dev/urandom
 260         default => 1024,
 261     },
 262     period => {
 263         type => 'integer',
 264         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 265             ." the guest to retrieve another 'max_bytes' of entropy.",
 266         optional => 1,
 267         default => 1000,
 268     },
 269 };
 270
 271 my $meta_info_fmt = {
 272     'ctime' => {
 273         type => 'integer',
 274         description => "The guest creation timestamp as UNIX epoch time",
 275         minimum => 0,
 276         optional => 1,
 277     },
 278     'creation-qemu' => {
 279         type => 'string',
 280         description => "The QEMU (machine) version from the time this VM was created.",
 281         pattern => '\d+(\.\d+)+',
 282         optional => 1,
 283     },
 284 };
 285
 286 my $confdesc = {
 287     onboot => {
 288         optional => 1,
 289         type => 'boolean',
 290         description => "Specifies whether a VM will be started during system bootup.",
 291         default => 0,
 292     },
 293     autostart => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Automatic restart after crash (currently ignored).",
 297         default => 0,
 298     },
 299     hotplug => {
 300         optional => 1,
 301         type => 'string', format => 'pve-hotplug-features',
 302         description => "Selectively enable hotplug features. This is a comma separated list of"
 303             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 304             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 305             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 306             ." windows > 7.",
 307         default => 'network,disk,usb',
 308     },
 309     reboot => {
 310         optional => 1,
 311         type => 'boolean',
 312         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 313         default => 1,
 314     },
 315     lock => {
 316         optional => 1,
 317         type => 'string',
 318         description => "Lock/unlock the VM.",
 319         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 320     },
 321     cpulimit => {
 322         optional => 1,
 323         type => 'number',
 324         description => "Limit of CPU usage.",
 325         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 326             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 327         minimum => 0,
 328         maximum => 128,
 329         default => 0,
 330     },
 331     cpuunits => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 335         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 336             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 337             ." weights of all the other running VMs.",
 338         minimum => 1,
 339         maximum => 262144,
 340         default => 'cgroup v1: 1024, cgroup v2: 100',
 341     },
 342     memory => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 346             ." you use the balloon device.",
 347         minimum => 16,
 348         default => 512,
 349     },
 350     balloon => {
 351         optional => 1,
 352         type => 'integer',
 353         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 354         minimum => 0,
 355     },
 356     shares => {
 357         optional => 1,
 358         type => 'integer',
 359         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 360             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 361             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 362         minimum => 0,
 363         maximum => 50000,
 364         default => 1000,
 365     },
 366     keyboard => {
 367         optional => 1,
 368         type => 'string',
 369         description => "Keyboard layout for VNC server. This option is generally not required and"
 370         ." is often better handled from within the guest OS.",
 371         enum => PVE::Tools::kvmkeymaplist(),
 372         default => undef,
 373     },
 374     name => {
 375         optional => 1,
 376         type => 'string', format => 'dns-name',
 377         description => "Set a name for the VM. Only used on the configuration web interface.",
 378     },
 379     scsihw => {
 380         optional => 1,
 381         type => 'string',
 382         description => "SCSI controller model",
 383         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 384         default => 'lsi',
 385     },
 386     description => {
 387         optional => 1,
 388         type => 'string',
 389         description => "Description for the VM. Shown in the web-interface VM's summary."
 390             ." This is saved as comment inside the configuration file.",
 391         maxLength => 1024 * 8,
 392     },
 393     ostype => {
 394         optional => 1,
 395         type => 'string',
 396         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 397         description => "Specify guest operating system.",
 398         verbose_description => <<EODESC,
 399 Specify guest operating system. This is used to enable special
 400 optimization/features for specific operating systems:
 401
 402 [horizontal]
 403 other;; unspecified OS
 404 wxp;; Microsoft Windows XP
 405 w2k;; Microsoft Windows 2000
 406 w2k3;; Microsoft Windows 2003
 407 w2k8;; Microsoft Windows 2008
 408 wvista;; Microsoft Windows Vista
 409 win7;; Microsoft Windows 7
 410 win8;; Microsoft Windows 8/2012/2012r2
 411 win10;; Microsoft Windows 10/2016/2019
 412 win11;; Microsoft Windows 11/2022
 413 l24;; Linux 2.4 Kernel
 414 l26;; Linux 2.6 - 6.X Kernel
 415 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 416 EODESC
 417     },
 418     boot => {
 419         optional => 1,
 420         type => 'string', format => 'pve-qm-boot',
 421         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 422             ." key or 'legacy=' is deprecated.",
 423     },
 424     bootdisk => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-bootdisk',
 427         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 428         pattern => '(ide|sata|scsi|virtio)\d+',
 429     },
 430     smp => {
 431         optional => 1,
 432         type => 'integer',
 433         description => "The number of CPUs. Please use option -sockets instead.",
 434         minimum => 1,
 435         default => 1,
 436     },
 437     sockets => {
 438         optional => 1,
 439         type => 'integer',
 440         description => "The number of CPU sockets.",
 441         minimum => 1,
 442         default => 1,
 443     },
 444     cores => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "The number of cores per socket.",
 448         minimum => 1,
 449         default => 1,
 450     },
 451     numa => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable NUMA.",
 455         default => 0,
 456     },
 457     hugepages => {
 458         optional => 1,
 459         type => 'string',
 460         description => "Enable/disable hugepages memory.",
 461         enum => [qw(any 2 1024)],
 462     },
 463     keephugepages => {
 464         optional => 1,
 465         type => 'boolean',
 466         default => 0,
 467         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 468             ." after VM shutdown and can be used for subsequent starts.",
 469     },
 470     vcpus => {
 471         optional => 1,
 472         type => 'integer',
 473         description => "Number of hotplugged vcpus.",
 474         minimum => 1,
 475         default => 0,
 476     },
 477     acpi => {
 478         optional => 1,
 479         type => 'boolean',
 480         description => "Enable/disable ACPI.",
 481         default => 1,
 482     },
 483     agent => {
 484         optional => 1,
 485         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 486         type => 'string',
 487         format => $agent_fmt,
 488     },
 489     kvm => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Enable/disable KVM hardware virtualization.",
 493         default => 1,
 494     },
 495     tdf => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable time drift fix.",
 499         default => 0,
 500     },
 501     localtime => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 505             ." the `ostype` indicates a Microsoft Windows OS.",
 506     },
 507     freeze => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 511     },
 512     vga => {
 513         optional => 1,
 514         type => 'string', format => $vga_fmt,
 515         description => "Configure the VGA hardware.",
 516         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 517             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 518             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 519             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 520             ." display server. For win* OS you can select how many independent displays you want,"
 521             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 522             ." using a serial device as terminal.",
 523     },
 524     watchdog => {
 525         optional => 1,
 526         type => 'string', format => 'pve-qm-watchdog',
 527         description => "Create a virtual hardware watchdog device.",
 528         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 529             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 530             ." else the watchdog will reset the guest (or execute the respective action specified)",
 531     },
 532     startdate => {
 533         optional => 1,
 534         type => 'string',
 535         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 536         description => "Set the initial date of the real time clock. Valid format for date are:"
 537             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 538         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 539         default => 'now',
 540     },
 541     startup =>  get_standard_option('pve-startup-order'),
 542     template => {
 543         optional => 1,
 544         type => 'boolean',
 545         description => "Enable/disable Template.",
 546         default => 0,
 547     },
 548     args => {
 549         optional => 1,
 550         type => 'string',
 551         description => "Arbitrary arguments passed to kvm.",
 552         verbose_description => <<EODESCR,
 553 Arbitrary arguments passed to kvm, for example:
 554
 555 args: -no-reboot -no-hpet
 556
 557 NOTE: this option is for experts only.
 558 EODESCR
 559     },
 560     tablet => {
 561         optional => 1,
 562         type => 'boolean',
 563         default => 1,
 564         description => "Enable/disable the USB tablet device.",
 565         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 566             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 567             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 568             ." may consider disabling this to save some context switches. This is turned off by"
 569             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 570     },
 571     migrate_speed => {
 572         optional => 1,
 573         type => 'integer',
 574         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 575         minimum => 0,
 576         default => 0,
 577     },
 578     migrate_downtime => {
 579         optional => 1,
 580         type => 'number',
 581         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 582         minimum => 0,
 583         default => 0.1,
 584     },
 585     cdrom => {
 586         optional => 1,
 587         type => 'string', format => 'pve-qm-ide',
 588         typetext => '<volume>',
 589         description => "This is an alias for option -ide2",
 590     },
 591     cpu => {
 592         optional => 1,
 593         description => "Emulated CPU type.",
 594         type => 'string',
 595         format => 'pve-vm-cpu-conf',
 596     },
 597     parent => get_standard_option('pve-snapshot-name', {
 598         optional => 1,
 599         description => "Parent snapshot name. This is used internally, and should not be modified.",
 600     }),
 601     snaptime => {
 602         optional => 1,
 603         description => "Timestamp for snapshots.",
 604         type => 'integer',
 605         minimum => 0,
 606     },
 607     vmstate => {
 608         optional => 1,
 609         type => 'string', format => 'pve-volume-id',
 610         description => "Reference to a volume which stores the VM state. This is used internally"
 611             ." for snapshots.",
 612     },
 613     vmstatestorage => get_standard_option('pve-storage-id', {
 614         description => "Default storage for VM state volumes/files.",
 615         optional => 1,
 616     }),
 617     runningmachine => get_standard_option('pve-qemu-machine', {
 618         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 619             ." for snapshots.",
 620     }),
 621     runningcpu => {
 622         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 623             ." internally for snapshots.",
 624         optional => 1,
 625         type => 'string',
 626         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 627         format_description => 'QEMU -cpu parameter'
 628     },
 629     machine => get_standard_option('pve-qemu-machine'),
 630     arch => {
 631         description => "Virtual processor architecture. Defaults to the host.",
 632         optional => 1,
 633         type => 'string',
 634         enum => [qw(x86_64 aarch64)],
 635     },
 636     smbios1 => {
 637         description => "Specify SMBIOS type 1 fields.",
 638         type => 'string', format => 'pve-qm-smbios1',
 639         maxLength => 512,
 640         optional => 1,
 641     },
 642     protection => {
 643         optional => 1,
 644         type => 'boolean',
 645         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 646             ." remove disk operations.",
 647         default => 0,
 648     },
 649     bios => {
 650         optional => 1,
 651         type => 'string',
 652         enum => [ qw(seabios ovmf) ],
 653         description => "Select BIOS implementation.",
 654         default => 'seabios',
 655     },
 656     vmgenid => {
 657         type => 'string',
 658         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 659         format_description => 'UUID',
 660         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 661             ." to disable explicitly.",
 662         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 663             ." value identifier to the guest OS. This allows to notify the guest operating system"
 664             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 665             ." execution or creation from a template). The guest operating system notices the"
 666             ." change, and is then able to react as appropriate by marking its copies of"
 667             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 668             ."Note that auto-creation only works when done through API/CLI create or update methods"
 669             .", but not when manually editing the config file.",
 670         default => "1 (autogenerated)",
 671         optional => 1,
 672     },
 673     hookscript => {
 674         type => 'string',
 675         format => 'pve-volume-id',
 676         optional => 1,
 677         description => "Script that will be executed during various steps in the vms lifetime.",
 678     },
 679     ivshmem => {
 680         type => 'string',
 681         format => $ivshmem_fmt,
 682         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 683             ." the host.",
 684         optional => 1,
 685     },
 686     audio0 => {
 687         type => 'string',
 688         format => $audio_fmt,
 689         description => "Configure a audio device, useful in combination with QXL/Spice.",
 690         optional => 1
 691     },
 692     spice_enhancements => {
 693         type => 'string',
 694         format => $spice_enhancements_fmt,
 695         description => "Configure additional enhancements for SPICE.",
 696         optional => 1
 697     },
 698     tags => {
 699         type => 'string', format => 'pve-tag-list',
 700         description => 'Tags of the VM. This is only meta information.',
 701         optional => 1,
 702     },
 703     rng0 => {
 704         type => 'string',
 705         format => $rng_fmt,
 706         description => "Configure a VirtIO-based Random Number Generator.",
 707         optional => 1,
 708     },
 709     meta => {
 710         type => 'string',
 711         format => $meta_info_fmt,
 712         description => "Some (read-only) meta-information about this guest.",
 713         optional => 1,
 714     },
 715     affinity => {
 716         type => 'string', format => 'pve-cpuset',
 717         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 718         optional => 1,
 719     },
 720 };
 721
 722 my $cicustom_fmt = {
 723     meta => {
 724         type => 'string',
 725         optional => 1,
 726         description => 'Specify a custom file containing all meta data passed to the VM via"
 727             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731     network => {
 732         type => 'string',
 733         optional => 1,
 734         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 735         format => 'pve-volume-id',
 736         format_description => 'volume',
 737     },
 738     user => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 742         format => 'pve-volume-id',
 743         format_description => 'volume',
 744     },
 745     vendor => {
 746         type => 'string',
 747         optional => 1,
 748         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 749         format => 'pve-volume-id',
 750         format_description => 'volume',
 751     },
 752 };
 753 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 754
 755 my $confdesc_cloudinit = {
 756     citype => {
 757         optional => 1,
 758         type => 'string',
 759         description => 'Specifies the cloud-init configuration format. The default depends on the'
 760             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 761             .' and `configdrive2` for windows.',
 762         enum => ['configdrive2', 'nocloud', 'opennebula'],
 763     },
 764     ciuser => {
 765         optional => 1,
 766         type => 'string',
 767         description => "cloud-init: User name to change ssh keys and password for instead of the"
 768             ." image's configured default user.",
 769     },
 770     cipassword => {
 771         optional => 1,
 772         type => 'string',
 773         description => 'cloud-init: Password to assign the user. Using this is generally not'
 774             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 775             .' support hashed passwords.',
 776     },
 777     cicustom => {
 778         optional => 1,
 779         type => 'string',
 780         description => 'cloud-init: Specify custom files to replace the automatically generated'
 781             .' ones at start.',
 782         format => 'pve-qm-cicustom',
 783     },
 784     searchdomain => {
 785         optional => 1,
 786         type => 'string',
 787         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 788             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 789             .' are set.',
 790     },
 791     nameserver => {
 792         optional => 1,
 793         type => 'string', format => 'address-list',
 794         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 795             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 796             .' are set.',
 797     },
 798     sshkeys => {
 799         optional => 1,
 800         type => 'string',
 801         format => 'urlencoded',
 802         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 803     },
 804 };
 805
 806 # what about other qemu settings ?
 807 #cpu => 'string',
 808 #machine => 'string',
 809 #fda => 'file',
 810 #fdb => 'file',
 811 #mtdblock => 'file',
 812 #sd => 'file',
 813 #pflash => 'file',
 814 #snapshot => 'bool',
 815 #bootp => 'file',
 816 ##tftp => 'dir',
 817 ##smb => 'dir',
 818 #kernel => 'file',
 819 #append => 'string',
 820 #initrd => 'file',
 821 ##soundhw => 'string',
 822
 823 while (my ($k, $v) = each %$confdesc) {
 824     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 825 }
 826
 827 my $MAX_USB_DEVICES = 14;
 828 my $MAX_NETS = 32;
 829 my $MAX_SERIAL_PORTS = 4;
 830 my $MAX_PARALLEL_PORTS = 3;
 831 my $MAX_NUMA = 8;
 832
 833 my $numa_fmt = {
 834     cpus => {
 835         type => "string",
 836         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 837         description => "CPUs accessing this NUMA node.",
 838         format_description => "id[-id];...",
 839     },
 840     memory => {
 841         type => "number",
 842         description => "Amount of memory this NUMA node provides.",
 843         optional => 1,
 844     },
 845     hostnodes => {
 846         type => "string",
 847         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 848         description => "Host NUMA nodes to use.",
 849         format_description => "id[-id];...",
 850         optional => 1,
 851     },
 852     policy => {
 853         type => 'string',
 854         enum => [qw(preferred bind interleave)],
 855         description => "NUMA allocation policy.",
 856         optional => 1,
 857     },
 858 };
 859 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 860 my $numadesc = {
 861     optional => 1,
 862     type => 'string', format => $numa_fmt,
 863     description => "NUMA topology.",
 864 };
 865 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 866
 867 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 868     $confdesc->{"numa$i"} = $numadesc;
 869 }
 870
 871 my $nic_model_list = [
 872     'e1000',
 873     'e1000-82540em',
 874     'e1000-82544gc',
 875     'e1000-82545em',
 876     'e1000e',
 877     'i82551',
 878     'i82557b',
 879     'i82559er',
 880     'ne2k_isa',
 881     'ne2k_pci',
 882     'pcnet',
 883     'rtl8139',
 884     'virtio',
 885     'vmxnet3',
 886 ];
 887 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 888
 889 my $net_fmt_bridge_descr = <<__EOD__;
 890 Bridge to attach the network device to. The Proxmox VE standard bridge
 891 is called 'vmbr0'.
 892
 893 If you do not specify a bridge, we create a kvm user (NATed) network
 894 device, which provides DHCP and DNS services. The following addresses
 895 are used:
 896
 897  10.0.2.2   Gateway
 898  10.0.2.3   DNS Server
 899  10.0.2.4   SMB Server
 900
 901 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 902 __EOD__
 903
 904 my $net_fmt = {
 905     macaddr  => get_standard_option('mac-addr', {
 906         description => "MAC address. That address must be unique withing your network. This is"
 907             ." automatically generated if not specified.",
 908     }),
 909     model => {
 910         type => 'string',
 911         description => "Network Card Model. The 'virtio' model provides the best performance with"
 912             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 913             ." best to use 'e1000'.",
 914         enum => $nic_model_list,
 915         default_key => 1,
 916     },
 917     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 918     bridge => get_standard_option('pve-bridge-id', {
 919         description => $net_fmt_bridge_descr,
 920         optional => 1,
 921     }),
 922     queues => {
 923         type => 'integer',
 924         minimum => 0, maximum => 64,
 925         description => 'Number of packet queues to be used on the device.',
 926         optional => 1,
 927     },
 928     rate => {
 929         type => 'number',
 930         minimum => 0,
 931         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 932         optional => 1,
 933     },
 934     tag => {
 935         type => 'integer',
 936         minimum => 1, maximum => 4094,
 937         description => 'VLAN tag to apply to packets on this interface.',
 938         optional => 1,
 939     },
 940     trunks => {
 941         type => 'string',
 942         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 943         description => 'VLAN trunks to pass through this interface.',
 944         format_description => 'vlanid[;vlanid...]',
 945         optional => 1,
 946     },
 947     firewall => {
 948         type => 'boolean',
 949         description => 'Whether this interface should be protected by the firewall.',
 950         optional => 1,
 951     },
 952     link_down => {
 953         type => 'boolean',
 954         description => 'Whether this interface should be disconnected (like pulling the plug).',
 955         optional => 1,
 956     },
 957     mtu => {
 958         type => 'integer',
 959         minimum => 1, maximum => 65520,
 960         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 961         optional => 1,
 962     },
 963 };
 964
 965 my $netdesc = {
 966     optional => 1,
 967     type => 'string', format => $net_fmt,
 968     description => "Specify network devices.",
 969 };
 970
 971 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 972
 973 my $ipconfig_fmt = {
 974     ip => {
 975         type => 'string',
 976         format => 'pve-ipv4-config',
 977         format_description => 'IPv4Format/CIDR',
 978         description => 'IPv4 address in CIDR format.',
 979         optional => 1,
 980         default => 'dhcp',
 981     },
 982     gw => {
 983         type => 'string',
 984         format => 'ipv4',
 985         format_description => 'GatewayIPv4',
 986         description => 'Default gateway for IPv4 traffic.',
 987         optional => 1,
 988         requires => 'ip',
 989     },
 990     ip6 => {
 991         type => 'string',
 992         format => 'pve-ipv6-config',
 993         format_description => 'IPv6Format/CIDR',
 994         description => 'IPv6 address in CIDR format.',
 995         optional => 1,
 996         default => 'dhcp',
 997     },
 998     gw6 => {
 999         type => 'string',
1000         format => 'ipv6',
1001         format_description => 'GatewayIPv6',
1002         description => 'Default gateway for IPv6 traffic.',
1003         optional => 1,
1004         requires => 'ip6',
1005     },
1006 };
1007 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1008 my $ipconfigdesc = {
1009     optional => 1,
1010     type => 'string', format => 'pve-qm-ipconfig',
1011     description => <<'EODESCR',
1012 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1013
1014 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1015
1016 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1017 gateway should be provided.
1018 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1019 cloud-init 19.4 or newer.
1020
1021 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1022 dhcp on IPv4.
1023 EODESCR
1024 };
1025 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1026
1027 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1028     $confdesc->{"net$i"} = $netdesc;
1029     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1030 }
1031
1032 foreach my $key (keys %$confdesc_cloudinit) {
1033     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1034 }
1035
1036 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1037 sub pve_verify_cpuset {
1038     my ($set_text, $noerr) = @_;
1039
1040     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1041
1042     if ($@) {
1043         return if $noerr;
1044         die "unable to parse cpuset option\n";
1045     }
1046
1047     return PVE::CpuSet->new($members)->short_string();
1048 }
1049
1050 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1051 sub verify_volume_id_or_qm_path {
1052     my ($volid, $noerr) = @_;
1053
1054     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1055
1056     return verify_volume_id_or_absolute_path($volid, $noerr);
1057 }
1058
1059 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1060 sub verify_volume_id_or_absolute_path {
1061     my ($volid, $noerr) = @_;
1062
1063     return $volid if $volid =~ m|^/|;
1064
1065     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1066     if ($@) {
1067         return if $noerr;
1068         die $@;
1069     }
1070     return $volid;
1071 }
1072
1073 my $usb_fmt = {
1074     host => {
1075         default_key => 1,
1076         type => 'string', format => 'pve-qm-usb-device',
1077         format_description => 'HOSTUSBDEVICE|spice',
1078         description => <<EODESCR,
1079 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1080
1081  'bus-port(.port)*' (decimal numbers) or
1082  'vendor_id:product_id' (hexadeciaml numbers) or
1083  'spice'
1084
1085 You can use the 'lsusb -t' command to list existing usb devices.
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1089
1090 The value 'spice' can be used to add a usb redirection devices for spice.
1091 EODESCR
1092     },
1093     usb3 => {
1094         optional => 1,
1095         type => 'boolean',
1096         description => "Specifies whether if given host option is a USB3 device or port."
1097             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1098             ." is irrelevant (all devices are plugged into a xhci controller).",
1099         default => 0,
1100     },
1101 };
1102
1103 my $usbdesc = {
1104     optional => 1,
1105     type => 'string', format => $usb_fmt,
1106     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1107         ." l26 or windows > 7, n can be up to 14).",
1108 };
1109 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1110
1111 my $serialdesc = {
1112         optional => 1,
1113         type => 'string',
1114         pattern => '(/dev/.+|socket)',
1115         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1116         verbose_description =>  <<EODESCR,
1117 Create a serial device inside the VM (n is 0 to 3), and pass through a
1118 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1119 host side (use 'qm terminal' to open a terminal connection).
1120
1121 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1122 use with special care.
1123
1124 CAUTION: Experimental! User reported problems with this option.
1125 EODESCR
1126 };
1127
1128 my $paralleldesc= {
1129         optional => 1,
1130         type => 'string',
1131         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1132         description =>  "Map host parallel devices (n is 0 to 2).",
1133         verbose_description =>  <<EODESCR,
1134 Map host parallel devices (n is 0 to 2).
1135
1136 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1137 machines - use with special care.
1138
1139 CAUTION: Experimental! User reported problems with this option.
1140 EODESCR
1141 };
1142
1143 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1144     $confdesc->{"parallel$i"} = $paralleldesc;
1145 }
1146
1147 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1148     $confdesc->{"serial$i"} = $serialdesc;
1149 }
1150
1151 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1152     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1153 }
1154
1155 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1156     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1157 }
1158
1159 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1160     $confdesc->{"usb$i"} = $usbdesc;
1161 }
1162
1163 my $boot_fmt = {
1164     legacy => {
1165         optional => 1,
1166         default_key => 1,
1167         type => 'string',
1168         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1169                      . " Deprecated, use 'order=' instead.",
1170         pattern => '[acdn]{1,4}',
1171         format_description => "[acdn]{1,4}",
1172
1173         # note: this is also the fallback if boot: is not given at all
1174         default => 'cdn',
1175     },
1176     order => {
1177         optional => 1,
1178         type => 'string',
1179         format => 'pve-qm-bootdev-list',
1180         format_description => "device[;device...]",
1181         description => <<EODESC,
1182 The guest will attempt to boot from devices in the order they appear here.
1183
1184 Disks, optical drives and passed-through storage USB devices will be directly
1185 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1186 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1187
1188 Note that only devices in this list will be marked as bootable and thus loaded
1189 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1190 (e.g. software-raid), you need to specify all of them here.
1191
1192 Overrides the deprecated 'legacy=[acdn]*' value when given.
1193 EODESC
1194     },
1195 };
1196 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1197
1198 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1199 sub verify_bootdev {
1200     my ($dev, $noerr) = @_;
1201
1202     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1203     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1204
1205     my $check = sub {
1206         my ($base) = @_;
1207         return 0 if $dev !~ m/^$base\d+$/;
1208         return 0 if !$confdesc->{$dev};
1209         return 1;
1210     };
1211
1212     return $dev if $check->("net");
1213     return $dev if $check->("usb");
1214     return $dev if $check->("hostpci");
1215
1216     return if $noerr;
1217     die "invalid boot device '$dev'\n";
1218 }
1219
1220 sub print_bootorder {
1221     my ($devs) = @_;
1222     return "" if !@$devs;
1223     my $data = { order => join(';', @$devs) };
1224     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1225 }
1226
1227 my $kvm_api_version = 0;
1228
1229 sub kvm_version {
1230     return $kvm_api_version if $kvm_api_version;
1231
1232     open my $fh, '<', '/dev/kvm' or return;
1233
1234     # 0xae00 => KVM_GET_API_VERSION
1235     $kvm_api_version = ioctl($fh, 0xae00, 0);
1236     close($fh);
1237
1238     return $kvm_api_version;
1239 }
1240
1241 my $kvm_user_version = {};
1242 my $kvm_mtime = {};
1243
1244 sub kvm_user_version {
1245     my ($binary) = @_;
1246
1247     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1248     my $st = stat($binary);
1249
1250     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1251     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1252         $cachedmtime == $st->mtime;
1253
1254     $kvm_user_version->{$binary} = 'unknown';
1255     $kvm_mtime->{$binary} = $st->mtime;
1256
1257     my $code = sub {
1258         my $line = shift;
1259         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1260             $kvm_user_version->{$binary} = $2;
1261         }
1262     };
1263
1264     eval { run_command([$binary, '--version'], outfunc => $code); };
1265     warn $@ if $@;
1266
1267     return $kvm_user_version->{$binary};
1268
1269 }
1270 my sub extract_version {
1271     my ($machine_type, $version) = @_;
1272     $version = kvm_user_version() if !defined($version);
1273     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1274 }
1275
1276 sub kernel_has_vhost_net {
1277     return -c '/dev/vhost-net';
1278 }
1279
1280 sub option_exists {
1281     my $key = shift;
1282     return defined($confdesc->{$key});
1283 }
1284
1285 my $cdrom_path;
1286 sub get_cdrom_path {
1287
1288     return $cdrom_path if defined($cdrom_path);
1289
1290     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1291
1292     if (!defined($cdrom_path)) {
1293         log_warn("no physical CD-ROM available, ignoring");
1294         $cdrom_path = '';
1295     }
1296
1297     return $cdrom_path;
1298 }
1299
1300 sub get_iso_path {
1301     my ($storecfg, $vmid, $cdrom) = @_;
1302
1303     if ($cdrom eq 'cdrom') {
1304         return get_cdrom_path();
1305     } elsif ($cdrom eq 'none') {
1306         return '';
1307     } elsif ($cdrom =~ m|^/|) {
1308         return $cdrom;
1309     } else {
1310         return PVE::Storage::path($storecfg, $cdrom);
1311     }
1312 }
1313
1314 # try to convert old style file names to volume IDs
1315 sub filename_to_volume_id {
1316     my ($vmid, $file, $media) = @_;
1317
1318      if (!($file eq 'none' || $file eq 'cdrom' ||
1319           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1320
1321         return if $file =~ m|/|;
1322
1323         if ($media && $media eq 'cdrom') {
1324             $file = "local:iso/$file";
1325         } else {
1326             $file = "local:$vmid/$file";
1327         }
1328     }
1329
1330     return $file;
1331 }
1332
1333 sub verify_media_type {
1334     my ($opt, $vtype, $media) = @_;
1335
1336     return if !$media;
1337
1338     my $etype;
1339     if ($media eq 'disk') {
1340         $etype = 'images';
1341     } elsif ($media eq 'cdrom') {
1342         $etype = 'iso';
1343     } else {
1344         die "internal error";
1345     }
1346
1347     return if ($vtype eq $etype);
1348
1349     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1350 }
1351
1352 sub cleanup_drive_path {
1353     my ($opt, $storecfg, $drive) = @_;
1354
1355     # try to convert filesystem paths to volume IDs
1356
1357     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1358         ($drive->{file} !~ m|^/dev/.+|) &&
1359         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1360         ($drive->{file} !~ m/^\d+$/)) {
1361         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1362         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1363             if !$vtype;
1364         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1365         verify_media_type($opt, $vtype, $drive->{media});
1366         $drive->{file} = $volid;
1367     }
1368
1369     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1370 }
1371
1372 sub parse_hotplug_features {
1373     my ($data) = @_;
1374
1375     my $res = {};
1376
1377     return $res if $data eq '0';
1378
1379     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1380
1381     foreach my $feature (PVE::Tools::split_list($data)) {
1382         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1383             $res->{$1} = 1;
1384         } else {
1385             die "invalid hotplug feature '$feature'\n";
1386         }
1387     }
1388     return $res;
1389 }
1390
1391 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1392 sub pve_verify_hotplug_features {
1393     my ($value, $noerr) = @_;
1394
1395     return $value if parse_hotplug_features($value);
1396
1397     return if $noerr;
1398
1399     die "unable to parse hotplug option\n";
1400 }
1401
1402 sub scsi_inquiry {
1403     my($fh, $noerr) = @_;
1404
1405     my $SG_IO = 0x2285;
1406     my $SG_GET_VERSION_NUM = 0x2282;
1407
1408     my $versionbuf = "\x00" x 8;
1409     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1410     if (!$ret) {
1411         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1412         return;
1413     }
1414     my $version = unpack("I", $versionbuf);
1415     if ($version < 30000) {
1416         die "scsi generic interface too old\n"  if !$noerr;
1417         return;
1418     }
1419
1420     my $buf = "\x00" x 36;
1421     my $sensebuf = "\x00" x 8;
1422     my $cmd = pack("C x3 C x1", 0x12, 36);
1423
1424     # see /usr/include/scsi/sg.h
1425     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1426
1427     my $packet = pack(
1428         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1429     );
1430
1431     $ret = ioctl($fh, $SG_IO, $packet);
1432     if (!$ret) {
1433         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1434         return;
1435     }
1436
1437     my @res = unpack($sg_io_hdr_t, $packet);
1438     if ($res[17] || $res[18]) {
1439         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1440         return;
1441     }
1442
1443     my $res = {};
1444     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1445
1446     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1447     $res->{type} &= 0x1F;
1448
1449     return $res;
1450 }
1451
1452 sub path_is_scsi {
1453     my ($path) = @_;
1454
1455     my $fh = IO::File->new("+<$path") || return;
1456     my $res = scsi_inquiry($fh, 1);
1457     close($fh);
1458
1459     return $res;
1460 }
1461
1462 sub print_tabletdevice_full {
1463     my ($conf, $arch) = @_;
1464
1465     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1466
1467     # we use uhci for old VMs because tablet driver was buggy in older qemu
1468     my $usbbus;
1469     if ($q35 || $arch eq 'aarch64') {
1470         $usbbus = 'ehci';
1471     } else {
1472         $usbbus = 'uhci';
1473     }
1474
1475     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1476 }
1477
1478 sub print_keyboarddevice_full {
1479     my ($conf, $arch) = @_;
1480
1481     return if $arch ne 'aarch64';
1482
1483     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1484 }
1485
1486 my sub get_drive_id {
1487     my ($drive) = @_;
1488     return "$drive->{interface}$drive->{index}";
1489 }
1490
1491 sub print_drivedevice_full {
1492     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1493
1494     my $device = '';
1495     my $maxdev = 0;
1496
1497     my $drive_id = get_drive_id($drive);
1498     if ($drive->{interface} eq 'virtio') {
1499         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1500         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1501         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1502     } elsif ($drive->{interface} eq 'scsi') {
1503
1504         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1505         my $unit = $drive->{index} % $maxdev;
1506         my $devicetype = 'hd';
1507         my $path = '';
1508         if (drive_is_cdrom($drive)) {
1509             $devicetype = 'cd';
1510         } else {
1511             if ($drive->{file} =~ m|^/|) {
1512                 $path = $drive->{file};
1513                 if (my $info = path_is_scsi($path)) {
1514                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1515                         $devicetype = 'block';
1516                     } elsif ($info->{type} == 1) { # tape
1517                         $devicetype = 'generic';
1518                     }
1519                 }
1520             } else {
1521                  $path = PVE::Storage::path($storecfg, $drive->{file});
1522             }
1523
1524             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1525             my $version = extract_version($machine_type, kvm_user_version());
1526             if ($path =~ m/^iscsi\:\/\// &&
1527                !min_version($version, 4, 1)) {
1528                 $devicetype = 'generic';
1529             }
1530         }
1531
1532         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1533             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1534         } else {
1535             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1536                 .",lun=$drive->{index}";
1537         }
1538         $device .= ",drive=drive-$drive_id,id=$drive_id";
1539
1540         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1541             $device .= ",rotation_rate=1";
1542         }
1543         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1544
1545     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1546         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1547         my $controller = int($drive->{index} / $maxdev);
1548         my $unit = $drive->{index} % $maxdev;
1549         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1550
1551         $device = "ide-$devicetype";
1552         if ($drive->{interface} eq 'ide') {
1553             $device .= ",bus=ide.$controller,unit=$unit";
1554         } else {
1555             $device .= ",bus=ahci$controller.$unit";
1556         }
1557         $device .= ",drive=drive-$drive_id,id=$drive_id";
1558
1559         if ($devicetype eq 'hd') {
1560             if (my $model = $drive->{model}) {
1561                 $model = URI::Escape::uri_unescape($model);
1562                 $device .= ",model=$model";
1563             }
1564             if ($drive->{ssd}) {
1565                 $device .= ",rotation_rate=1";
1566             }
1567         }
1568         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1569     } elsif ($drive->{interface} eq 'usb') {
1570         die "implement me";
1571         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1572     } else {
1573         die "unsupported interface type";
1574     }
1575
1576     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1577
1578     if (my $serial = $drive->{serial}) {
1579         $serial = URI::Escape::uri_unescape($serial);
1580         $device .= ",serial=$serial";
1581     }
1582
1583
1584     return $device;
1585 }
1586
1587 sub get_initiator_name {
1588     my $initiator;
1589
1590     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1591     while (defined(my $line = <$fh>)) {
1592         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1593         $initiator = $1;
1594         last;
1595     }
1596     $fh->close();
1597
1598     return $initiator;
1599 }
1600
1601 my sub storage_allows_io_uring_default {
1602     my ($scfg, $cache_direct) = @_;
1603
1604     # io_uring with cache mode writeback or writethrough on krbd will hang...
1605     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1606
1607     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1608     # sometimes, just plain disable...
1609     return if $scfg && $scfg->{type} eq 'lvm';
1610
1611     # io_uring causes problems when used with CIFS since kernel 5.15
1612     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1613     return if $scfg && $scfg->{type} eq 'cifs';
1614
1615     return 1;
1616 }
1617
1618 my sub drive_uses_cache_direct {
1619     my ($drive, $scfg) = @_;
1620
1621     my $cache_direct = 0;
1622
1623     if (my $cache = $drive->{cache}) {
1624         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1625     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1626         $cache_direct = 1;
1627     }
1628
1629     return $cache_direct;
1630 }
1631
1632 sub print_drive_commandline_full {
1633     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1634
1635     my $path;
1636     my $volid = $drive->{file};
1637     my $format = $drive->{format};
1638     my $drive_id = get_drive_id($drive);
1639
1640     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1641     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1642
1643     if (drive_is_cdrom($drive)) {
1644         $path = get_iso_path($storecfg, $vmid, $volid);
1645         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1646     } else {
1647         if ($storeid) {
1648             $path = PVE::Storage::path($storecfg, $volid);
1649             $format //= qemu_img_format($scfg, $volname);
1650         } else {
1651             $path = $volid;
1652             $format //= "raw";
1653         }
1654    }
1655
1656    my $is_rbd = $path =~ m/^rbd:/;
1657
1658     my $opts = '';
1659     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1660     foreach my $o (@qemu_drive_options) {
1661         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1662     }
1663
1664     # snapshot only accepts on|off
1665     if (defined($drive->{snapshot})) {
1666         my $v = $drive->{snapshot} ? 'on' : 'off';
1667         $opts .= ",snapshot=$v";
1668     }
1669
1670     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1671         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1672     }
1673
1674     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1675         my ($dir, $qmpname) = @$type;
1676         if (my $v = $drive->{"mbps$dir"}) {
1677             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1678         }
1679         if (my $v = $drive->{"mbps${dir}_max"}) {
1680             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1681         }
1682         if (my $v = $drive->{"bps${dir}_max_length"}) {
1683             $opts .= ",throttling.bps$qmpname-max-length=$v";
1684         }
1685         if (my $v = $drive->{"iops${dir}"}) {
1686             $opts .= ",throttling.iops$qmpname=$v";
1687         }
1688         if (my $v = $drive->{"iops${dir}_max"}) {
1689             $opts .= ",throttling.iops$qmpname-max=$v";
1690         }
1691         if (my $v = $drive->{"iops${dir}_max_length"}) {
1692             $opts .= ",throttling.iops$qmpname-max-length=$v";
1693         }
1694     }
1695
1696     if ($pbs_name) {
1697         $format = "rbd" if $is_rbd;
1698         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1699             if !$format;
1700         $opts .= ",format=alloc-track,file.driver=$format";
1701     } elsif ($format) {
1702         $opts .= ",format=$format";
1703     }
1704
1705     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1706
1707     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1708
1709     if (!$drive->{aio}) {
1710         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1711             # io_uring supports all cache modes
1712             $opts .= ",aio=io_uring";
1713         } else {
1714             # aio native works only with O_DIRECT
1715             if($cache_direct) {
1716                 $opts .= ",aio=native";
1717             } else {
1718                 $opts .= ",aio=threads";
1719             }
1720         }
1721     }
1722
1723     if (!drive_is_cdrom($drive)) {
1724         my $detectzeroes;
1725         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1726             $detectzeroes = 'off';
1727         } elsif ($drive->{discard}) {
1728             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1729         } else {
1730             # This used to be our default with discard not being specified:
1731             $detectzeroes = 'on';
1732         }
1733
1734         # note: 'detect-zeroes' works per blockdev and we want it to persist
1735         # after the alloc-track is removed, so put it on 'file' directly
1736         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1737         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1738     }
1739
1740     if ($pbs_name) {
1741         $opts .= ",backing=$pbs_name";
1742         $opts .= ",auto-remove=on";
1743     }
1744
1745     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1746     my $file_param = "file";
1747     if ($pbs_name) {
1748         # non-rbd drivers require the underlying file to be a seperate block
1749         # node, so add a second .file indirection
1750         $file_param .= ".file" if !$is_rbd;
1751         $file_param .= ".filename";
1752     }
1753     my $pathinfo = $path ? "$file_param=$path," : '';
1754
1755     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1756 }
1757
1758 sub print_pbs_blockdev {
1759     my ($pbs_conf, $pbs_name) = @_;
1760     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1761     $blockdev .= ",repository=$pbs_conf->{repository}";
1762     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1763     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1764     $blockdev .= ",archive=$pbs_conf->{archive}";
1765     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1766     return $blockdev;
1767 }
1768
1769 sub print_netdevice_full {
1770     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1771
1772     my $device = $net->{model};
1773     if ($net->{model} eq 'virtio') {
1774          $device = 'virtio-net-pci';
1775      };
1776
1777     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1778     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1779     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1780         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1781         # and out of each queue plus one config interrupt and control vector queue
1782         my $vectors = $net->{queues} * 2 + 2;
1783         $tmpstr .= ",vectors=$vectors,mq=on";
1784         if (min_version($machine_version, 7, 1)) {
1785             $tmpstr .= ",packed=on";
1786         }
1787     }
1788
1789     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1790         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1791     }
1792
1793     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1794
1795     if (my $mtu = $net->{mtu}) {
1796         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1797             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1798             if ($mtu == 1) {
1799                  $mtu = $bridge_mtu;
1800             } elsif ($mtu < 576) {
1801                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1802             } elsif ($mtu > $bridge_mtu) {
1803                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1804             }
1805             $tmpstr .= ",host_mtu=$mtu";
1806         } else {
1807             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1808         }
1809     }
1810
1811     if ($use_old_bios_files) {
1812         my $romfile;
1813         if ($device eq 'virtio-net-pci') {
1814             $romfile = 'pxe-virtio.rom';
1815         } elsif ($device eq 'e1000') {
1816             $romfile = 'pxe-e1000.rom';
1817         } elsif ($device eq 'e1000e') {
1818             $romfile = 'pxe-e1000e.rom';
1819         } elsif ($device eq 'ne2k') {
1820             $romfile = 'pxe-ne2k_pci.rom';
1821         } elsif ($device eq 'pcnet') {
1822             $romfile = 'pxe-pcnet.rom';
1823         } elsif ($device eq 'rtl8139') {
1824             $romfile = 'pxe-rtl8139.rom';
1825         }
1826         $tmpstr .= ",romfile=$romfile" if $romfile;
1827     }
1828
1829     return $tmpstr;
1830 }
1831
1832 sub print_netdev_full {
1833     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1834
1835     my $i = '';
1836     if ($netid =~ m/^net(\d+)$/) {
1837         $i = int($1);
1838     }
1839
1840     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1841
1842     my $ifname = "tap${vmid}i$i";
1843
1844     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1845     die "interface name '$ifname' is too long (max 15 character)\n"
1846         if length($ifname) >= 16;
1847
1848     my $vhostparam = '';
1849     if (is_native($arch)) {
1850         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1851     }
1852
1853     my $vmname = $conf->{name} || "vm$vmid";
1854
1855     my $netdev = "";
1856     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1857
1858     if ($net->{bridge}) {
1859         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1860             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1861     } else {
1862         $netdev = "type=user,id=$netid,hostname=$vmname";
1863     }
1864
1865     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1866
1867     return $netdev;
1868 }
1869
1870 my $vga_map = {
1871     'cirrus' => 'cirrus-vga',
1872     'std' => 'VGA',
1873     'vmware' => 'vmware-svga',
1874     'virtio' => 'virtio-vga',
1875     'virtio-gl' => 'virtio-vga-gl',
1876 };
1877
1878 sub print_vga_device {
1879     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1880
1881     my $type = $vga_map->{$vga->{type}};
1882     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1883         $type = 'virtio-gpu';
1884     }
1885     my $vgamem_mb = $vga->{memory};
1886
1887     my $max_outputs = '';
1888     if ($qxlnum) {
1889         $type = $id ? 'qxl' : 'qxl-vga';
1890
1891         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1892             # set max outputs so linux can have up to 4 qxl displays with one device
1893             if (min_version($machine_version, 4, 1)) {
1894                 $max_outputs = ",max_outputs=4";
1895             }
1896         }
1897     }
1898
1899     die "no devicetype for $vga->{type}\n" if !$type;
1900
1901     my $memory = "";
1902     if ($vgamem_mb) {
1903         if ($vga->{type} =~ /^virtio/) {
1904             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1905             $memory = ",max_hostmem=$bytes";
1906         } elsif ($qxlnum) {
1907             # from https://www.spice-space.org/multiple-monitors.html
1908             $memory = ",vgamem_mb=$vga->{memory}";
1909             my $ram = $vgamem_mb * 4;
1910             my $vram = $vgamem_mb * 2;
1911             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1912         } else {
1913             $memory = ",vgamem_mb=$vga->{memory}";
1914         }
1915     } elsif ($qxlnum && $id) {
1916         $memory = ",ram_size=67108864,vram_size=33554432";
1917     }
1918
1919     my $edidoff = "";
1920     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1921         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1922     }
1923
1924     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1925     my $vgaid = "vga" . ($id // '');
1926     my $pciaddr;
1927     if ($q35 && $vgaid eq 'vga') {
1928         # the first display uses pcie.0 bus on q35 machines
1929         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1930     } else {
1931         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1932     }
1933
1934     if ($vga->{type} eq 'virtio-gl') {
1935         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1936         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1937             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1938
1939         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1940             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1941     }
1942
1943     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1944 }
1945
1946 sub parse_number_sets {
1947     my ($set) = @_;
1948     my $res = [];
1949     foreach my $part (split(/;/, $set)) {
1950         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1951             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1952             push @$res, [ $1, $2 ];
1953         } else {
1954             die "invalid range: $part\n";
1955         }
1956     }
1957     return $res;
1958 }
1959
1960 sub parse_numa {
1961     my ($data) = @_;
1962
1963     my $res = parse_property_string($numa_fmt, $data);
1964     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1965     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1966     return $res;
1967 }
1968
1969 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1970 sub parse_net {
1971     my ($data, $disable_mac_autogen) = @_;
1972
1973     my $res = eval { parse_property_string($net_fmt, $data) };
1974     if ($@) {
1975         warn $@;
1976         return;
1977     }
1978     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1979         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1980         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1981     }
1982     return $res;
1983 }
1984
1985 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1986 sub parse_ipconfig {
1987     my ($data) = @_;
1988
1989     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1990     if ($@) {
1991         warn $@;
1992         return;
1993     }
1994
1995     if ($res->{gw} && !$res->{ip}) {
1996         warn 'gateway specified without specifying an IP address';
1997         return;
1998     }
1999     if ($res->{gw6} && !$res->{ip6}) {
2000         warn 'IPv6 gateway specified without specifying an IPv6 address';
2001         return;
2002     }
2003     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2004         warn 'gateway specified together with DHCP';
2005         return;
2006     }
2007     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2008         # gw6 + auto/dhcp
2009         warn "IPv6 gateway specified together with $res->{ip6} address";
2010         return;
2011     }
2012
2013     if (!$res->{ip} && !$res->{ip6}) {
2014         return { ip => 'dhcp', ip6 => 'dhcp' };
2015     }
2016
2017     return $res;
2018 }
2019
2020 sub print_net {
2021     my $net = shift;
2022
2023     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2024 }
2025
2026 sub add_random_macs {
2027     my ($settings) = @_;
2028
2029     foreach my $opt (keys %$settings) {
2030         next if $opt !~ m/^net(\d+)$/;
2031         my $net = parse_net($settings->{$opt});
2032         next if !$net;
2033         $settings->{$opt} = print_net($net);
2034     }
2035 }
2036
2037 sub vm_is_volid_owner {
2038     my ($storecfg, $vmid, $volid) = @_;
2039
2040     if ($volid !~  m|^/|) {
2041         my ($path, $owner);
2042         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2043         if ($owner && ($owner == $vmid)) {
2044             return 1;
2045         }
2046     }
2047
2048     return;
2049 }
2050
2051 sub vmconfig_register_unused_drive {
2052     my ($storecfg, $vmid, $conf, $drive) = @_;
2053
2054     if (drive_is_cloudinit($drive)) {
2055         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2056         warn $@ if $@;
2057         delete $conf->{cloudinit};
2058     } elsif (!drive_is_cdrom($drive)) {
2059         my $volid = $drive->{file};
2060         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2061             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2062         }
2063     }
2064 }
2065
2066 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2067 my $smbios1_fmt = {
2068     uuid => {
2069         type => 'string',
2070         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2071         format_description => 'UUID',
2072         description => "Set SMBIOS1 UUID.",
2073         optional => 1,
2074     },
2075     version => {
2076         type => 'string',
2077         pattern => '[A-Za-z0-9+\/]+={0,2}',
2078         format_description => 'Base64 encoded string',
2079         description => "Set SMBIOS1 version.",
2080         optional => 1,
2081     },
2082     serial => {
2083         type => 'string',
2084         pattern => '[A-Za-z0-9+\/]+={0,2}',
2085         format_description => 'Base64 encoded string',
2086         description => "Set SMBIOS1 serial number.",
2087         optional => 1,
2088     },
2089     manufacturer => {
2090         type => 'string',
2091         pattern => '[A-Za-z0-9+\/]+={0,2}',
2092         format_description => 'Base64 encoded string',
2093         description => "Set SMBIOS1 manufacturer.",
2094         optional => 1,
2095     },
2096     product => {
2097         type => 'string',
2098         pattern => '[A-Za-z0-9+\/]+={0,2}',
2099         format_description => 'Base64 encoded string',
2100         description => "Set SMBIOS1 product ID.",
2101         optional => 1,
2102     },
2103     sku => {
2104         type => 'string',
2105         pattern => '[A-Za-z0-9+\/]+={0,2}',
2106         format_description => 'Base64 encoded string',
2107         description => "Set SMBIOS1 SKU string.",
2108         optional => 1,
2109     },
2110     family => {
2111         type => 'string',
2112         pattern => '[A-Za-z0-9+\/]+={0,2}',
2113         format_description => 'Base64 encoded string',
2114         description => "Set SMBIOS1 family string.",
2115         optional => 1,
2116     },
2117     base64 => {
2118         type => 'boolean',
2119         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2120         optional => 1,
2121     },
2122 };
2123
2124 sub parse_smbios1 {
2125     my ($data) = @_;
2126
2127     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2128     warn $@ if $@;
2129     return $res;
2130 }
2131
2132 sub print_smbios1 {
2133     my ($smbios1) = @_;
2134     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2135 }
2136
2137 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2138
2139 sub parse_watchdog {
2140     my ($value) = @_;
2141
2142     return if !$value;
2143
2144     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2145     warn $@ if $@;
2146     return $res;
2147 }
2148
2149 sub parse_guest_agent {
2150     my ($conf) = @_;
2151
2152     return {} if !defined($conf->{agent});
2153
2154     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2155     warn $@ if $@;
2156
2157     # if the agent is disabled ignore the other potentially set properties
2158     return {} if !$res->{enabled};
2159     return $res;
2160 }
2161
2162 sub get_qga_key {
2163     my ($conf, $key) = @_;
2164     return undef if !defined($conf->{agent});
2165
2166     my $agent = parse_guest_agent($conf);
2167     return $agent->{$key};
2168 }
2169
2170 sub parse_vga {
2171     my ($value) = @_;
2172
2173     return {} if !$value;
2174     my $res = eval { parse_property_string($vga_fmt, $value) };
2175     warn $@ if $@;
2176     return $res;
2177 }
2178
2179 sub parse_rng {
2180     my ($value) = @_;
2181
2182     return if !$value;
2183
2184     my $res = eval { parse_property_string($rng_fmt, $value) };
2185     warn $@ if $@;
2186     return $res;
2187 }
2188
2189 sub parse_meta_info {
2190     my ($value) = @_;
2191
2192     return if !$value;
2193
2194     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2195     warn $@ if $@;
2196     return $res;
2197 }
2198
2199 sub new_meta_info_string {
2200     my () = @_; # for now do not allow to override any value
2201
2202     return PVE::JSONSchema::print_property_string(
2203         {
2204             'creation-qemu' => kvm_user_version(),
2205             ctime => "". int(time()),
2206         },
2207         $meta_info_fmt
2208     );
2209 }
2210
2211 sub qemu_created_version_fixups {
2212     my ($conf, $forcemachine, $kvmver) = @_;
2213
2214     my $meta = parse_meta_info($conf->{meta}) // {};
2215     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2216
2217     # check if we need to apply some handling for VMs that always use the latest machine version but
2218     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2219     # would be required (we do not want to pin machine version for non-windows OS type)
2220     if (
2221         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2222         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2223         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2224         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2225     ) {
2226         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2227         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2228             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2229             # and thus with the predictable interface naming of systemd
2230             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2231         }
2232     }
2233     return;
2234 }
2235
2236 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2237 sub verify_usb_device {
2238     my ($value, $noerr) = @_;
2239
2240     return $value if parse_usb_device($value);
2241
2242     return if $noerr;
2243
2244     die "unable to parse usb device\n";
2245 }
2246
2247 # add JSON properties for create and set function
2248 sub json_config_properties {
2249     my ($prop, $with_disk_alloc) = @_;
2250
2251     my $skip_json_config_opts = {
2252         parent => 1,
2253         snaptime => 1,
2254         vmstate => 1,
2255         runningmachine => 1,
2256         runningcpu => 1,
2257         meta => 1,
2258     };
2259
2260     foreach my $opt (keys %$confdesc) {
2261         next if $skip_json_config_opts->{$opt};
2262
2263         if ($with_disk_alloc && is_valid_drivename($opt)) {
2264             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2265         } else {
2266             $prop->{$opt} = $confdesc->{$opt};
2267         }
2268     }
2269
2270     return $prop;
2271 }
2272
2273 # Properties that we can read from an OVF file
2274 sub json_ovf_properties {
2275     my $prop = {};
2276
2277     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2278         $prop->{$device} = {
2279             type => 'string',
2280             format => 'pve-volume-id-or-absolute-path',
2281             description => "Disk image that gets imported to $device",
2282             optional => 1,
2283         };
2284     }
2285
2286     $prop->{cores} = {
2287         type => 'integer',
2288         description => "The number of CPU cores.",
2289         optional => 1,
2290     };
2291     $prop->{memory} = {
2292         type => 'integer',
2293         description => "Amount of RAM for the VM in MB.",
2294         optional => 1,
2295     };
2296     $prop->{name} = {
2297         type => 'string',
2298         description => "Name of the VM.",
2299         optional => 1,
2300     };
2301
2302     return $prop;
2303 }
2304
2305 # return copy of $confdesc_cloudinit to generate documentation
2306 sub cloudinit_config_properties {
2307
2308     return dclone($confdesc_cloudinit);
2309 }
2310
2311 sub cloudinit_pending_properties {
2312     my $p = {
2313         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2314         name => 1,
2315     };
2316     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2317     return $p;
2318 }
2319
2320 sub check_type {
2321     my ($key, $value) = @_;
2322
2323     die "unknown setting '$key'\n" if !$confdesc->{$key};
2324
2325     my $type = $confdesc->{$key}->{type};
2326
2327     if (!defined($value)) {
2328         die "got undefined value\n";
2329     }
2330
2331     if ($value =~ m/[\n\r]/) {
2332         die "property contains a line feed\n";
2333     }
2334
2335     if ($type eq 'boolean') {
2336         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2337         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2338         die "type check ('boolean') failed - got '$value'\n";
2339     } elsif ($type eq 'integer') {
2340         return int($1) if $value =~ m/^(\d+)$/;
2341         die "type check ('integer') failed - got '$value'\n";
2342     } elsif ($type eq 'number') {
2343         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2344         die "type check ('number') failed - got '$value'\n";
2345     } elsif ($type eq 'string') {
2346         if (my $fmt = $confdesc->{$key}->{format}) {
2347             PVE::JSONSchema::check_format($fmt, $value);
2348             return $value;
2349         }
2350         $value =~ s/^\"(.*)\"$/$1/;
2351         return $value;
2352     } else {
2353         die "internal error"
2354     }
2355 }
2356
2357 sub destroy_vm {
2358     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2359
2360     my $conf = PVE::QemuConfig->load_config($vmid);
2361
2362     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2363         PVE::QemuConfig->check_lock($conf);
2364     }
2365
2366     if ($conf->{template}) {
2367         # check if any base image is still used by a linked clone
2368         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2369                 my ($ds, $drive) = @_;
2370                 return if drive_is_cdrom($drive);
2371
2372                 my $volid = $drive->{file};
2373                 return if !$volid || $volid =~ m|^/|;
2374
2375                 die "base volume '$volid' is still in use by linked cloned\n"
2376                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2377
2378         });
2379     }
2380
2381     my $volids = {};
2382     my $remove_owned_drive = sub {
2383         my ($ds, $drive) = @_;
2384         return if drive_is_cdrom($drive, 1);
2385
2386         my $volid = $drive->{file};
2387         return if !$volid || $volid =~ m|^/|;
2388         return if $volids->{$volid};
2389
2390         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2391         return if !$path || !$owner || ($owner != $vmid);
2392
2393         $volids->{$volid} = 1;
2394         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2395         warn "Could not remove disk '$volid', check manually: $@" if $@;
2396     };
2397
2398     # only remove disks owned by this VM (referenced in the config)
2399     my $include_opts = {
2400         include_unused => 1,
2401         extra_keys => ['vmstate'],
2402     };
2403     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2404
2405     for my $snap (values %{$conf->{snapshots}}) {
2406         next if !defined($snap->{vmstate});
2407         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2408         next if !defined($drive);
2409         $remove_owned_drive->('vmstate', $drive);
2410     }
2411
2412     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2413
2414     if ($purge_unreferenced) { # also remove unreferenced disk
2415         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2416         PVE::Storage::foreach_volid($vmdisks, sub {
2417             my ($volid, $sid, $volname, $d) = @_;
2418             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2419             warn $@ if $@;
2420         });
2421     }
2422
2423     if (defined $replacement_conf) {
2424         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2425     } else {
2426         PVE::QemuConfig->destroy_config($vmid);
2427     }
2428 }
2429
2430 sub parse_vm_config {
2431     my ($filename, $raw, $strict) = @_;
2432
2433     return if !defined($raw);
2434
2435     my $res = {
2436         digest => Digest::SHA::sha1_hex($raw),
2437         snapshots => {},
2438         pending => {},
2439         cloudinit => {},
2440     };
2441
2442     my $handle_error = sub {
2443         my ($msg) = @_;
2444
2445         if ($strict) {
2446             die $msg;
2447         } else {
2448             warn $msg;
2449         }
2450     };
2451
2452     $filename =~ m|/qemu-server/(\d+)\.conf$|
2453         || die "got strange filename '$filename'";
2454
2455     my $vmid = $1;
2456
2457     my $conf = $res;
2458     my $descr;
2459     my $finish_description = sub {
2460         if (defined($descr)) {
2461             $descr =~ s/\s+$//;
2462             $conf->{description} = $descr;
2463         }
2464         $descr = undef;
2465     };
2466     my $section = '';
2467
2468     my @lines = split(/\n/, $raw);
2469     foreach my $line (@lines) {
2470         next if $line =~ m/^\s*$/;
2471
2472         if ($line =~ m/^\[PENDING\]\s*$/i) {
2473             $section = 'pending';
2474             $finish_description->();
2475             $conf = $res->{$section} = {};
2476             next;
2477         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2478             $section = 'cloudinit';
2479             $finish_description->();
2480             $conf = $res->{$section} = {};
2481             next;
2482
2483         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2484             $section = $1;
2485             $finish_description->();
2486             $conf = $res->{snapshots}->{$section} = {};
2487             next;
2488         }
2489
2490         if ($line =~ m/^\#(.*)$/) {
2491             $descr = '' if !defined($descr);
2492             $descr .= PVE::Tools::decode_text($1) . "\n";
2493             next;
2494         }
2495
2496         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2497             $descr = '' if !defined($descr);
2498             $descr .= PVE::Tools::decode_text($2);
2499         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2500             $conf->{snapstate} = $1;
2501         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2502             my $key = $1;
2503             my $value = $2;
2504             $conf->{$key} = $value;
2505         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2506             my $value = $1;
2507             if ($section eq 'pending') {
2508                 $conf->{delete} = $value; # we parse this later
2509             } else {
2510                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2511             }
2512         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2513             my $key = $1;
2514             my $value = $2;
2515             if ($section eq 'cloudinit') {
2516                 # ignore validation only used for informative purpose
2517                 $conf->{$key} = $value;
2518                 next;
2519             }
2520             eval { $value = check_type($key, $value); };
2521             if ($@) {
2522                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2523             } else {
2524                 $key = 'ide2' if $key eq 'cdrom';
2525                 my $fmt = $confdesc->{$key}->{format};
2526                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2527                     my $v = parse_drive($key, $value);
2528                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2529                         $v->{file} = $volid;
2530                         $value = print_drive($v);
2531                     } else {
2532                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2533                         next;
2534                     }
2535                 }
2536
2537                 $conf->{$key} = $value;
2538             }
2539         } else {
2540             $handle_error->("vm $vmid - unable to parse config: $line\n");
2541         }
2542     }
2543
2544     $finish_description->();
2545     delete $res->{snapstate}; # just to be sure
2546
2547     return $res;
2548 }
2549
2550 sub write_vm_config {
2551     my ($filename, $conf) = @_;
2552
2553     delete $conf->{snapstate}; # just to be sure
2554
2555     if ($conf->{cdrom}) {
2556         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2557         $conf->{ide2} = $conf->{cdrom};
2558         delete $conf->{cdrom};
2559     }
2560
2561     # we do not use 'smp' any longer
2562     if ($conf->{sockets}) {
2563         delete $conf->{smp};
2564     } elsif ($conf->{smp}) {
2565         $conf->{sockets} = $conf->{smp};
2566         delete $conf->{cores};
2567         delete $conf->{smp};
2568     }
2569
2570     my $used_volids = {};
2571
2572     my $cleanup_config = sub {
2573         my ($cref, $pending, $snapname) = @_;
2574
2575         foreach my $key (keys %$cref) {
2576             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2577                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2578             my $value = $cref->{$key};
2579             if ($key eq 'delete') {
2580                 die "propertry 'delete' is only allowed in [PENDING]\n"
2581                     if !$pending;
2582                 # fixme: check syntax?
2583                 next;
2584             }
2585             eval { $value = check_type($key, $value); };
2586             die "unable to parse value of '$key' - $@" if $@;
2587
2588             $cref->{$key} = $value;
2589
2590             if (!$snapname && is_valid_drivename($key)) {
2591                 my $drive = parse_drive($key, $value);
2592                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2593             }
2594         }
2595     };
2596
2597     &$cleanup_config($conf);
2598
2599     &$cleanup_config($conf->{pending}, 1);
2600
2601     foreach my $snapname (keys %{$conf->{snapshots}}) {
2602         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2603         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2604     }
2605
2606     # remove 'unusedX' settings if we re-add a volume
2607     foreach my $key (keys %$conf) {
2608         my $value = $conf->{$key};
2609         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2610             delete $conf->{$key};
2611         }
2612     }
2613
2614     my $generate_raw_config = sub {
2615         my ($conf, $pending) = @_;
2616
2617         my $raw = '';
2618
2619         # add description as comment to top of file
2620         if (defined(my $descr = $conf->{description})) {
2621             if ($descr) {
2622                 foreach my $cl (split(/\n/, $descr)) {
2623                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2624                 }
2625             } else {
2626                 $raw .= "#\n" if $pending;
2627             }
2628         }
2629
2630         foreach my $key (sort keys %$conf) {
2631             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2632             $raw .= "$key: $conf->{$key}\n";
2633         }
2634         return $raw;
2635     };
2636
2637     my $raw = &$generate_raw_config($conf);
2638
2639     if (scalar(keys %{$conf->{pending}})){
2640         $raw .= "\n[PENDING]\n";
2641         $raw .= &$generate_raw_config($conf->{pending}, 1);
2642     }
2643
2644     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2645         $raw .= "\n[special:cloudinit]\n";
2646         $raw .= &$generate_raw_config($conf->{cloudinit});
2647     }
2648
2649     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2650         $raw .= "\n[$snapname]\n";
2651         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2652     }
2653
2654     return $raw;
2655 }
2656
2657 sub load_defaults {
2658
2659     my $res = {};
2660
2661     # we use static defaults from our JSON schema configuration
2662     foreach my $key (keys %$confdesc) {
2663         if (defined(my $default = $confdesc->{$key}->{default})) {
2664             $res->{$key} = $default;
2665         }
2666     }
2667
2668     return $res;
2669 }
2670
2671 sub config_list {
2672     my $vmlist = PVE::Cluster::get_vmlist();
2673     my $res = {};
2674     return $res if !$vmlist || !$vmlist->{ids};
2675     my $ids = $vmlist->{ids};
2676     my $nodename = nodename();
2677
2678     foreach my $vmid (keys %$ids) {
2679         my $d = $ids->{$vmid};
2680         next if !$d->{node} || $d->{node} ne $nodename;
2681         next if !$d->{type} || $d->{type} ne 'qemu';
2682         $res->{$vmid}->{exists} = 1;
2683     }
2684     return $res;
2685 }
2686
2687 # test if VM uses local resources (to prevent migration)
2688 sub check_local_resources {
2689     my ($conf, $noerr) = @_;
2690
2691     my @loc_res = ();
2692
2693     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2694     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2695
2696     push @loc_res, "ivshmem" if $conf->{ivshmem};
2697
2698     foreach my $k (keys %$conf) {
2699         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2700         # sockets are safe: they will recreated be on the target side post-migrate
2701         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2702         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2703     }
2704
2705     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2706
2707     return \@loc_res;
2708 }
2709
2710 # check if used storages are available on all nodes (use by migrate)
2711 sub check_storage_availability {
2712     my ($storecfg, $conf, $node) = @_;
2713
2714     PVE::QemuConfig->foreach_volume($conf, sub {
2715         my ($ds, $drive) = @_;
2716
2717         my $volid = $drive->{file};
2718         return if !$volid;
2719
2720         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2721         return if !$sid;
2722
2723         # check if storage is available on both nodes
2724         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2725         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2726
2727         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2728
2729         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2730             if !$scfg->{content}->{$vtype};
2731    });
2732 }
2733
2734 # list nodes where all VM images are available (used by has_feature API)
2735 sub shared_nodes {
2736     my ($conf, $storecfg) = @_;
2737
2738     my $nodelist = PVE::Cluster::get_nodelist();
2739     my $nodehash = { map { $_ => 1 } @$nodelist };
2740     my $nodename = nodename();
2741
2742     PVE::QemuConfig->foreach_volume($conf, sub {
2743         my ($ds, $drive) = @_;
2744
2745         my $volid = $drive->{file};
2746         return if !$volid;
2747
2748         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2749         if ($storeid) {
2750             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2751             if ($scfg->{disable}) {
2752                 $nodehash = {};
2753             } elsif (my $avail = $scfg->{nodes}) {
2754                 foreach my $node (keys %$nodehash) {
2755                     delete $nodehash->{$node} if !$avail->{$node};
2756                 }
2757             } elsif (!$scfg->{shared}) {
2758                 foreach my $node (keys %$nodehash) {
2759                     delete $nodehash->{$node} if $node ne $nodename
2760                 }
2761             }
2762         }
2763     });
2764
2765     return $nodehash
2766 }
2767
2768 sub check_local_storage_availability {
2769     my ($conf, $storecfg) = @_;
2770
2771     my $nodelist = PVE::Cluster::get_nodelist();
2772     my $nodehash = { map { $_ => {} } @$nodelist };
2773
2774     PVE::QemuConfig->foreach_volume($conf, sub {
2775         my ($ds, $drive) = @_;
2776
2777         my $volid = $drive->{file};
2778         return if !$volid;
2779
2780         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2781         if ($storeid) {
2782             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2783
2784             if ($scfg->{disable}) {
2785                 foreach my $node (keys %$nodehash) {
2786                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2787                 }
2788             } elsif (my $avail = $scfg->{nodes}) {
2789                 foreach my $node (keys %$nodehash) {
2790                     if (!$avail->{$node}) {
2791                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2792                     }
2793                 }
2794             }
2795         }
2796     });
2797
2798     foreach my $node (values %$nodehash) {
2799         if (my $unavail = $node->{unavailable_storages}) {
2800             $node->{unavailable_storages} = [ sort keys %$unavail ];
2801         }
2802     }
2803
2804     return $nodehash
2805 }
2806
2807 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2808 sub check_running {
2809     my ($vmid, $nocheck, $node) = @_;
2810
2811     # $nocheck is set when called during a migration, in which case the config
2812     # file might still or already reside on the *other* node
2813     # - because rename has already happened, and current node is source
2814     # - because rename hasn't happened yet, and current node is target
2815     # - because rename has happened, current node is target, but hasn't yet
2816     # processed it yet
2817     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2818     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2819 }
2820
2821 sub vzlist {
2822
2823     my $vzlist = config_list();
2824
2825     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2826
2827     while (defined(my $de = $fd->read)) {
2828         next if $de !~ m/^(\d+)\.pid$/;
2829         my $vmid = $1;
2830         next if !defined($vzlist->{$vmid});
2831         if (my $pid = check_running($vmid)) {
2832             $vzlist->{$vmid}->{pid} = $pid;
2833         }
2834     }
2835
2836     return $vzlist;
2837 }
2838
2839 our $vmstatus_return_properties = {
2840     vmid => get_standard_option('pve-vmid'),
2841     status => {
2842         description => "QEMU process status.",
2843         type => 'string',
2844         enum => ['stopped', 'running'],
2845     },
2846     maxmem => {
2847         description => "Maximum memory in bytes.",
2848         type => 'integer',
2849         optional => 1,
2850         renderer => 'bytes',
2851     },
2852     maxdisk => {
2853         description => "Root disk size in bytes.",
2854         type => 'integer',
2855         optional => 1,
2856         renderer => 'bytes',
2857     },
2858     name => {
2859         description => "VM name.",
2860         type => 'string',
2861         optional => 1,
2862     },
2863     qmpstatus => {
2864         description => "QEMU QMP agent status.",
2865         type => 'string',
2866         optional => 1,
2867     },
2868     pid => {
2869         description => "PID of running qemu process.",
2870         type => 'integer',
2871         optional => 1,
2872     },
2873     uptime => {
2874         description => "Uptime.",
2875         type => 'integer',
2876         optional => 1,
2877         renderer => 'duration',
2878     },
2879     cpus => {
2880         description => "Maximum usable CPUs.",
2881         type => 'number',
2882         optional => 1,
2883     },
2884     lock => {
2885         description => "The current config lock, if any.",
2886         type => 'string',
2887         optional => 1,
2888     },
2889     tags => {
2890         description  => "The current configured tags, if any",
2891         type => 'string',
2892         optional => 1,
2893     },
2894     'running-machine' => {
2895         description => "The currently running machine type (if running).",
2896         type => 'string',
2897         optional => 1,
2898     },
2899     'running-qemu' => {
2900         description => "The currently running QEMU version (if running).",
2901         type => 'string',
2902         optional => 1,
2903     },
2904 };
2905
2906 my $last_proc_pid_stat;
2907
2908 # get VM status information
2909 # This must be fast and should not block ($full == false)
2910 # We only query KVM using QMP if $full == true (this can be slow)
2911 sub vmstatus {
2912     my ($opt_vmid, $full) = @_;
2913
2914     my $res = {};
2915
2916     my $storecfg = PVE::Storage::config();
2917
2918     my $list = vzlist();
2919     my $defaults = load_defaults();
2920
2921     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2922
2923     my $cpucount = $cpuinfo->{cpus} || 1;
2924
2925     foreach my $vmid (keys %$list) {
2926         next if $opt_vmid && ($vmid ne $opt_vmid);
2927
2928         my $conf = PVE::QemuConfig->load_config($vmid);
2929
2930         my $d = { vmid => int($vmid) };
2931         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2932
2933         # fixme: better status?
2934         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2935
2936         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2937         if (defined($size)) {
2938             $d->{disk} = 0; # no info available
2939             $d->{maxdisk} = $size;
2940         } else {
2941             $d->{disk} = 0;
2942             $d->{maxdisk} = 0;
2943         }
2944
2945         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2946             * ($conf->{cores} || $defaults->{cores});
2947         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2948         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2949
2950         $d->{name} = $conf->{name} || "VM $vmid";
2951         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2952             : $defaults->{memory}*(1024*1024);
2953
2954         if ($conf->{balloon}) {
2955             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2956             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2957                 : $defaults->{shares};
2958         }
2959
2960         $d->{uptime} = 0;
2961         $d->{cpu} = 0;
2962         $d->{mem} = 0;
2963
2964         $d->{netout} = 0;
2965         $d->{netin} = 0;
2966
2967         $d->{diskread} = 0;
2968         $d->{diskwrite} = 0;
2969
2970         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2971
2972         $d->{serial} = 1 if conf_has_serial($conf);
2973         $d->{lock} = $conf->{lock} if $conf->{lock};
2974         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2975
2976         $res->{$vmid} = $d;
2977     }
2978
2979     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2980     foreach my $dev (keys %$netdev) {
2981         next if $dev !~ m/^tap([1-9]\d*)i/;
2982         my $vmid = $1;
2983         my $d = $res->{$vmid};
2984         next if !$d;
2985
2986         $d->{netout} += $netdev->{$dev}->{receive};
2987         $d->{netin} += $netdev->{$dev}->{transmit};
2988
2989         if ($full) {
2990             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2991             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2992         }
2993
2994     }
2995
2996     my $ctime = gettimeofday;
2997
2998     foreach my $vmid (keys %$list) {
2999
3000         my $d = $res->{$vmid};
3001         my $pid = $d->{pid};
3002         next if !$pid;
3003
3004         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3005         next if !$pstat; # not running
3006
3007         my $used = $pstat->{utime} + $pstat->{stime};
3008
3009         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3010
3011         if ($pstat->{vsize}) {
3012             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3013         }
3014
3015         my $old = $last_proc_pid_stat->{$pid};
3016         if (!$old) {
3017             $last_proc_pid_stat->{$pid} = {
3018                 time => $ctime,
3019                 used => $used,
3020                 cpu => 0,
3021             };
3022             next;
3023         }
3024
3025         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3026
3027         if ($dtime > 1000) {
3028             my $dutime = $used -  $old->{used};
3029
3030             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3031             $last_proc_pid_stat->{$pid} = {
3032                 time => $ctime,
3033                 used => $used,
3034                 cpu => $d->{cpu},
3035             };
3036         } else {
3037             $d->{cpu} = $old->{cpu};
3038         }
3039     }
3040
3041     return $res if !$full;
3042
3043     my $qmpclient = PVE::QMPClient->new();
3044
3045     my $ballooncb = sub {
3046         my ($vmid, $resp) = @_;
3047
3048         my $info = $resp->{'return'};
3049         return if !$info->{max_mem};
3050
3051         my $d = $res->{$vmid};
3052
3053         # use memory assigned to VM
3054         $d->{maxmem} = $info->{max_mem};
3055         $d->{balloon} = $info->{actual};
3056
3057         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3058             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3059             $d->{freemem} = $info->{free_mem};
3060         }
3061
3062         $d->{ballooninfo} = $info;
3063     };
3064
3065     my $blockstatscb = sub {
3066         my ($vmid, $resp) = @_;
3067         my $data = $resp->{'return'} || [];
3068         my $totalrdbytes = 0;
3069         my $totalwrbytes = 0;
3070
3071         for my $blockstat (@$data) {
3072             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3073             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3074
3075             $blockstat->{device} =~ s/drive-//;
3076             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3077         }
3078         $res->{$vmid}->{diskread} = $totalrdbytes;
3079         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3080     };
3081
3082     my $machinecb = sub {
3083         my ($vmid, $resp) = @_;
3084         my $data = $resp->{'return'} || [];
3085
3086         $res->{$vmid}->{'running-machine'} =
3087             PVE::QemuServer::Machine::current_from_query_machines($data);
3088     };
3089
3090     my $versioncb = sub {
3091         my ($vmid, $resp) = @_;
3092         my $data = $resp->{'return'} // {};
3093         my $version = 'unknown';
3094
3095         if (my $v = $data->{qemu}) {
3096             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3097         }
3098
3099         $res->{$vmid}->{'running-qemu'} = $version;
3100     };
3101
3102     my $statuscb = sub {
3103         my ($vmid, $resp) = @_;
3104
3105         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3106         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3107         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3108         # this fails if ballon driver is not loaded, so this must be
3109         # the last commnand (following command are aborted if this fails).
3110         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3111
3112         my $status = 'unknown';
3113         if (!defined($status = $resp->{'return'}->{status})) {
3114             warn "unable to get VM status\n";
3115             return;
3116         }
3117
3118         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3119     };
3120
3121     foreach my $vmid (keys %$list) {
3122         next if $opt_vmid && ($vmid ne $opt_vmid);
3123         next if !$res->{$vmid}->{pid}; # not running
3124         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3125     }
3126
3127     $qmpclient->queue_execute(undef, 2);
3128
3129     foreach my $vmid (keys %$list) {
3130         next if $opt_vmid && ($vmid ne $opt_vmid);
3131         next if !$res->{$vmid}->{pid}; #not running
3132
3133         # we can't use the $qmpclient since it might have already aborted on
3134         # 'query-balloon', but this might also fail for older versions...
3135         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3136         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3137     }
3138
3139     foreach my $vmid (keys %$list) {
3140         next if $opt_vmid && ($vmid ne $opt_vmid);
3141         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3142     }
3143
3144     return $res;
3145 }
3146
3147 sub conf_has_serial {
3148     my ($conf) = @_;
3149
3150     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3151         if ($conf->{"serial$i"}) {
3152             return 1;
3153         }
3154     }
3155
3156     return 0;
3157 }
3158
3159 sub conf_has_audio {
3160     my ($conf, $id) = @_;
3161
3162     $id //= 0;
3163     my $audio = $conf->{"audio$id"};
3164     return if !defined($audio);
3165
3166     my $audioproperties = parse_property_string($audio_fmt, $audio);
3167     my $audiodriver = $audioproperties->{driver} // 'spice';
3168
3169     return {
3170         dev => $audioproperties->{device},
3171         dev_id => "audiodev$id",
3172         backend => $audiodriver,
3173         backend_id => "$audiodriver-backend${id}",
3174     };
3175 }
3176
3177 sub audio_devs {
3178     my ($audio, $audiopciaddr, $machine_version) = @_;
3179
3180     my $devs = [];
3181
3182     my $id = $audio->{dev_id};
3183     my $audiodev = "";
3184     if (min_version($machine_version, 4, 2)) {
3185         $audiodev = ",audiodev=$audio->{backend_id}";
3186     }
3187
3188     if ($audio->{dev} eq 'AC97') {
3189         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3190     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3191         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3192         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3193         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3194     } else {
3195         die "unkown audio device '$audio->{dev}', implement me!";
3196     }
3197
3198     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3199
3200     return $devs;
3201 }
3202
3203 sub get_tpm_paths {
3204     my ($vmid) = @_;
3205     return {
3206         socket => "/var/run/qemu-server/$vmid.swtpm",
3207         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3208     };
3209 }
3210
3211 sub add_tpm_device {
3212     my ($vmid, $devices, $conf) = @_;
3213
3214     return if !$conf->{tpmstate0};
3215
3216     my $paths = get_tpm_paths($vmid);
3217
3218     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3219     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3220     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3221 }
3222
3223 sub start_swtpm {
3224     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3225
3226     return if !$tpmdrive;
3227
3228     my $state;
3229     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3230     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3231     if ($storeid) {
3232         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3233     } else {
3234         $state = $tpm->{file};
3235     }
3236
3237     my $paths = get_tpm_paths($vmid);
3238
3239     # during migration, we will get state from remote
3240     #
3241     if (!$migration) {
3242         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3243         my $setup_cmd = [
3244             "swtpm_setup",
3245             "--tpmstate",
3246             "file://$state",
3247             "--createek",
3248             "--create-ek-cert",
3249             "--create-platform-cert",
3250             "--lock-nvram",
3251             "--config",
3252             "/etc/swtpm_setup.conf", # do not use XDG configs
3253             "--runas",
3254             "0", # force creation as root, error if not possible
3255             "--not-overwrite", # ignore existing state, do not modify
3256         ];
3257
3258         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3259         # TPM 2.0 supports ECC crypto, use if possible
3260         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3261
3262         run_command($setup_cmd, outfunc => sub {
3263             print "swtpm_setup: $1\n";
3264         });
3265     }
3266
3267     my $emulator_cmd = [
3268         "swtpm",
3269         "socket",
3270         "--tpmstate",
3271         "backend-uri=file://$state,mode=0600",
3272         "--ctrl",
3273         "type=unixio,path=$paths->{socket},mode=0600",
3274         "--pid",
3275         "file=$paths->{pid}",
3276         "--terminate", # terminate on QEMU disconnect
3277         "--daemon",
3278     ];
3279     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3280     run_command($emulator_cmd, outfunc => sub { print $1; });
3281
3282     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3283     while (! -e $paths->{pid}) {
3284         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3285         usleep(50_000);
3286     }
3287
3288     # return untainted PID of swtpm daemon so it can be killed on error
3289     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3290     return $1;
3291 }
3292
3293 sub vga_conf_has_spice {
3294     my ($vga) = @_;
3295
3296     my $vgaconf = parse_vga($vga);
3297     my $vgatype = $vgaconf->{type};
3298     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3299
3300     return $1 || 1;
3301 }
3302
3303 sub is_native($) {
3304     my ($arch) = @_;
3305     return get_host_arch() eq $arch;
3306 }
3307
3308 sub get_vm_arch {
3309     my ($conf) = @_;
3310     return $conf->{arch} // get_host_arch();
3311 }
3312
3313 my $default_machines = {
3314     x86_64 => 'pc',
3315     aarch64 => 'virt',
3316 };
3317
3318 sub get_installed_machine_version {
3319     my ($kvmversion) = @_;
3320     $kvmversion = kvm_user_version() if !defined($kvmversion);
3321     $kvmversion =~ m/^(\d+\.\d+)/;
3322     return $1;
3323 }
3324
3325 sub windows_get_pinned_machine_version {
3326     my ($machine, $base_version, $kvmversion) = @_;
3327
3328     my $pin_version = $base_version;
3329     if (!defined($base_version) ||
3330         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3331     ) {
3332         $pin_version = get_installed_machine_version($kvmversion);
3333     }
3334     if (!$machine || $machine eq 'pc') {
3335         $machine = "pc-i440fx-$pin_version";
3336     } elsif ($machine eq 'q35') {
3337         $machine = "pc-q35-$pin_version";
3338     } elsif ($machine eq 'virt') {
3339         $machine = "virt-$pin_version";
3340     } else {
3341         warn "unknown machine type '$machine', not touching that!\n";
3342     }
3343
3344     return $machine;
3345 }
3346
3347 sub get_vm_machine {
3348     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3349
3350     my $machine = $forcemachine || $conf->{machine};
3351
3352     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3353         $kvmversion //= kvm_user_version();
3354         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3355         # layout which confuses windows quite a bit and may result in various regressions..
3356         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3357         if (windows_version($conf->{ostype})) {
3358             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3359         }
3360         $arch //= 'x86_64';
3361         $machine ||= $default_machines->{$arch};
3362         if ($add_pve_version) {
3363             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3364             $machine .= "+pve$pvever";
3365         }
3366     }
3367
3368     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3369         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3370         $machine = $1 if $is_pxe;
3371
3372         # for version-pinned machines that do not include a pve-version (e.g.
3373         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3374         $machine .= '+pve0';
3375
3376         $machine .= '.pxe' if $is_pxe;
3377     }
3378
3379     return $machine;
3380 }
3381
3382 sub get_ovmf_files($$$) {
3383     my ($arch, $efidisk, $smm) = @_;
3384
3385     my $types = $OVMF->{$arch}
3386         or die "no OVMF images known for architecture '$arch'\n";
3387
3388     my $type = 'default';
3389     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3390         $type = $smm ? "4m" : "4m-no-smm";
3391         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3392     }
3393
3394     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3395     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3396     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3397
3398     return ($ovmf_code, $ovmf_vars);
3399 }
3400
3401 my $Arch2Qemu = {
3402     aarch64 => '/usr/bin/qemu-system-aarch64',
3403     x86_64 => '/usr/bin/qemu-system-x86_64',
3404 };
3405 sub get_command_for_arch($) {
3406     my ($arch) = @_;
3407     return '/usr/bin/kvm' if is_native($arch);
3408
3409     my $cmd = $Arch2Qemu->{$arch}
3410         or die "don't know how to emulate architecture '$arch'\n";
3411     return $cmd;
3412 }
3413
3414 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3415 # to use in a QEMU command line (-cpu element), first array_intersect the result
3416 # of query_supported_ with query_understood_. This is necessary because:
3417 #
3418 # a) query_understood_ returns flags the host cannot use and
3419 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3420 #    flags, but CPU settings - with most of them being flags. Those settings
3421 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3422 #
3423 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3424 # expensive. If you need the value returned from this, you can get it much
3425 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3426 # $accel being 'kvm' or 'tcg'.
3427 #
3428 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3429 # changes, automatically populating pmxcfs.
3430 #
3431 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3432 # since kvm and tcg machines support different flags
3433 #
3434 sub query_supported_cpu_flags {
3435     my ($arch) = @_;
3436
3437     $arch //= get_host_arch();
3438     my $default_machine = $default_machines->{$arch};
3439
3440     my $flags = {};
3441
3442     # FIXME: Once this is merged, the code below should work for ARM as well:
3443     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3444     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3445         $arch eq "aarch64";
3446
3447     my $kvm_supported = defined(kvm_version());
3448     my $qemu_cmd = get_command_for_arch($arch);
3449     my $fakevmid = -1;
3450     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3451
3452     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3453     my $query_supported_run_qemu = sub {
3454         my ($kvm) = @_;
3455
3456         my $flags = {};
3457         my $cmd = [
3458             $qemu_cmd,
3459             '-machine', $default_machine,
3460             '-display', 'none',
3461             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3462             '-mon', 'chardev=qmp,mode=control',
3463             '-pidfile', $pidfile,
3464             '-S', '-daemonize'
3465         ];
3466
3467         if (!$kvm) {
3468             push @$cmd, '-accel', 'tcg';
3469         }
3470
3471         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3472         die "QEMU flag querying VM exited with code " . $rc if $rc;
3473
3474         eval {
3475             my $cmd_result = mon_cmd(
3476                 $fakevmid,
3477                 'query-cpu-model-expansion',
3478                 type => 'full',
3479                 model => { name => 'host' }
3480             );
3481
3482             my $props = $cmd_result->{model}->{props};
3483             foreach my $prop (keys %$props) {
3484                 next if $props->{$prop} ne '1';
3485                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3486                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3487                 # We only keep those with underscores, to match /proc/cpuinfo
3488                 $prop =~ s/\.|-/_/g;
3489                 $flags->{$prop} = 1;
3490             }
3491         };
3492         my $err = $@;
3493
3494         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3495         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3496
3497         die $err if $err;
3498
3499         return [ sort keys %$flags ];
3500     };
3501
3502     # We need to query QEMU twice, since KVM and TCG have different supported flags
3503     PVE::QemuConfig->lock_config($fakevmid, sub {
3504         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3505         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3506
3507         if ($kvm_supported) {
3508             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3509             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3510         }
3511     });
3512
3513     return $flags;
3514 }
3515
3516 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3517 my $understood_cpu_flag_dir = "/usr/share/kvm";
3518 sub query_understood_cpu_flags {
3519     my $arch = get_host_arch();
3520     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3521
3522     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3523         if ! -e $filepath;
3524
3525     my $raw = file_get_contents($filepath);
3526     $raw =~ s/^\s+|\s+$//g;
3527     my @flags = split(/\s+/, $raw);
3528
3529     return \@flags;
3530 }
3531
3532 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3533 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3534 my sub should_disable_smm {
3535     my ($conf, $vga) = @_;
3536
3537     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3538         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3539 }
3540
3541 my sub print_ovmf_drive_commandlines {
3542     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3543
3544     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3545
3546     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3547
3548     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3549     if ($d) {
3550         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3551         my ($path, $format) = $d->@{'file', 'format'};
3552         if ($storeid) {
3553             $path = PVE::Storage::path($storecfg, $d->{file});
3554             if (!defined($format)) {
3555                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3556                 $format = qemu_img_format($scfg, $volname);
3557             }
3558         } elsif (!defined($format)) {
3559             die "efidisk format must be specified\n";
3560         }
3561         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3562         if ($path =~ m/^rbd:/) {
3563             $var_drive_str .= ',cache=writeback';
3564             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3565         }
3566         $var_drive_str .= ",format=$format,file=$path";
3567
3568         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3569         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3570     } else {
3571         log_warn("no efidisk configured! Using temporary efivars disk.");
3572         my $path = "/tmp/$vmid-ovmf.fd";
3573         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3574         $var_drive_str .= ",format=raw,file=$path";
3575         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3576     }
3577
3578     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3579 }
3580
3581 sub config_to_command {
3582     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3583         $pbs_backing) = @_;
3584
3585     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3586     my $devices = [];
3587     my $bridges = {};
3588     my $ostype = $conf->{ostype};
3589     my $winversion = windows_version($ostype);
3590     my $kvm = $conf->{kvm};
3591     my $nodename = nodename();
3592
3593     my $arch = get_vm_arch($conf);
3594     my $kvm_binary = get_command_for_arch($arch);
3595     my $kvmver = kvm_user_version($kvm_binary);
3596
3597     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3598         $kvmver //= "undefined";
3599         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3600     }
3601
3602     my $add_pve_version = min_version($kvmver, 4, 1);
3603
3604     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3605     my $machine_version = extract_version($machine_type, $kvmver);
3606     $kvm //= 1 if is_native($arch);
3607
3608     $machine_version =~ m/(\d+)\.(\d+)/;
3609     my ($machine_major, $machine_minor) = ($1, $2);
3610
3611     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3612         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3613     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3614         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3615             ." please upgrade node '$nodename'\n"
3616     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3617         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3618         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3619             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3620             ." node '$nodename'\n";
3621     }
3622
3623     # if a specific +pve version is required for a feature, use $version_guard
3624     # instead of min_version to allow machines to be run with the minimum
3625     # required version
3626     my $required_pve_version = 0;
3627     my $version_guard = sub {
3628         my ($major, $minor, $pve) = @_;
3629         return 0 if !min_version($machine_version, $major, $minor, $pve);
3630         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3631         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3632         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3633         return 1;
3634     };
3635
3636     if ($kvm && !defined kvm_version()) {
3637         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3638             ." or enable in BIOS.\n";
3639     }
3640
3641     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3642     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3643     my $use_old_bios_files = undef;
3644     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3645
3646     my $cmd = [];
3647     if ($conf->{affinity}) {
3648         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3649     }
3650
3651     push @$cmd, $kvm_binary;
3652
3653     push @$cmd, '-id', $vmid;
3654
3655     my $vmname = $conf->{name} || "vm$vmid";
3656
3657     push @$cmd, '-name', "$vmname,debug-threads=on";
3658
3659     push @$cmd, '-no-shutdown';
3660
3661     my $use_virtio = 0;
3662
3663     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3664     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3665     push @$cmd, '-mon', "chardev=qmp,mode=control";
3666
3667     if (min_version($machine_version, 2, 12)) {
3668         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3669         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3670     }
3671
3672     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3673
3674     push @$cmd, '-daemonize';
3675
3676     if ($conf->{smbios1}) {
3677         my $smbios_conf = parse_smbios1($conf->{smbios1});
3678         if ($smbios_conf->{base64}) {
3679             # Do not pass base64 flag to qemu
3680             delete $smbios_conf->{base64};
3681             my $smbios_string = "";
3682             foreach my $key (keys %$smbios_conf) {
3683                 my $value;
3684                 if ($key eq "uuid") {
3685                     $value = $smbios_conf->{uuid}
3686                 } else {
3687                     $value = decode_base64($smbios_conf->{$key});
3688                 }
3689                 # qemu accepts any binary data, only commas need escaping by double comma
3690                 $value =~ s/,/,,/g;
3691                 $smbios_string .= "," . $key . "=" . $value if $value;
3692             }
3693             push @$cmd, '-smbios', "type=1" . $smbios_string;
3694         } else {
3695             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3696         }
3697     }
3698
3699     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3700         my ($code_drive_str, $var_drive_str) =
3701             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3702         push $cmd->@*, '-drive', $code_drive_str;
3703         push $cmd->@*, '-drive', $var_drive_str;
3704     }
3705
3706     if ($q35) { # tell QEMU to load q35 config early
3707         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3708         if (min_version($machine_version, 4, 0)) {
3709             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3710         } else {
3711             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3712         }
3713     }
3714
3715     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3716         push @$cmd, $fixups->@*;
3717     }
3718
3719     if ($conf->{vmgenid}) {
3720         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3721     }
3722
3723     # add usb controllers
3724     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3725         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3726     push @$devices, @usbcontrollers if @usbcontrollers;
3727     my $vga = parse_vga($conf->{vga});
3728
3729     my $qxlnum = vga_conf_has_spice($conf->{vga});
3730     $vga->{type} = 'qxl' if $qxlnum;
3731
3732     if (!$vga->{type}) {
3733         if ($arch eq 'aarch64') {
3734             $vga->{type} = 'virtio';
3735         } elsif (min_version($machine_version, 2, 9)) {
3736             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3737         } else {
3738             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3739         }
3740     }
3741
3742     # enable absolute mouse coordinates (needed by vnc)
3743     my $tablet = $conf->{tablet};
3744     if (!defined($tablet)) {
3745         $tablet = $defaults->{tablet};
3746         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3747         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3748     }
3749
3750     if ($tablet) {
3751         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3752         my $kbd = print_keyboarddevice_full($conf, $arch);
3753         push @$devices, '-device', $kbd if defined($kbd);
3754     }
3755
3756     my $bootorder = device_bootorder($conf);
3757
3758     # host pci device passthrough
3759     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3760         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3761
3762     # usb devices
3763     my $usb_dev_features = {};
3764     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3765
3766     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3767         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3768     push @$devices, @usbdevices if @usbdevices;
3769
3770     # serial devices
3771     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3772         my $path = $conf->{"serial$i"} or next;
3773         if ($path eq 'socket') {
3774             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3775             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3776             # On aarch64, serial0 is the UART device. QEMU only allows
3777             # connecting UART devices via the '-serial' command line, as
3778             # the device has a fixed slot on the hardware...
3779             if ($arch eq 'aarch64' && $i == 0) {
3780                 push @$devices, '-serial', "chardev:serial$i";
3781             } else {
3782                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3783             }
3784         } else {
3785             die "no such serial device\n" if ! -c $path;
3786             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3787             push @$devices, '-device', "isa-serial,chardev=serial$i";
3788         }
3789     }
3790
3791     # parallel devices
3792     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3793         if (my $path = $conf->{"parallel$i"}) {
3794             die "no such parallel device\n" if ! -c $path;
3795             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3796             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3797             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3798         }
3799     }
3800
3801     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3802         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3803         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3804         push @$devices, @$audio_devs;
3805     }
3806
3807     add_tpm_device($vmid, $devices, $conf);
3808
3809     my $sockets = 1;
3810     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3811     $sockets = $conf->{sockets} if  $conf->{sockets};
3812
3813     my $cores = $conf->{cores} || 1;
3814
3815     my $maxcpus = $sockets * $cores;
3816
3817     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3818
3819     my $allowed_vcpus = $cpuinfo->{cpus};
3820
3821     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3822
3823     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3824         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3825         for (my $i = 2; $i <= $vcpus; $i++)  {
3826             my $cpustr = print_cpu_device($conf,$i);
3827             push @$cmd, '-device', $cpustr;
3828         }
3829
3830     } else {
3831
3832         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3833     }
3834     push @$cmd, '-nodefaults';
3835
3836     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3837
3838     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3839
3840     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3841
3842     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3843         push @$devices, '-device', print_vga_device(
3844             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3845
3846         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3847
3848         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3849         push @$cmd,  '-vnc', "unix:$socket,password=on";
3850     } else {
3851         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3852         push @$cmd, '-nographic';
3853     }
3854
3855     # time drift fix
3856     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3857     my $useLocaltime = $conf->{localtime};
3858
3859     if ($winversion >= 5) { # windows
3860         $useLocaltime = 1 if !defined($conf->{localtime});
3861
3862         # use time drift fix when acpi is enabled
3863         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3864             $tdf = 1 if !defined($conf->{tdf});
3865         }
3866     }
3867
3868     if ($winversion >= 6) {
3869         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3870         push @$cmd, '-no-hpet';
3871     }
3872
3873     push @$rtcFlags, 'driftfix=slew' if $tdf;
3874
3875     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3876         push @$rtcFlags, "base=$conf->{startdate}";
3877     } elsif ($useLocaltime) {
3878         push @$rtcFlags, 'base=localtime';
3879     }
3880
3881     if ($forcecpu) {
3882         push @$cmd, '-cpu', $forcecpu;
3883     } else {
3884         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3885     }
3886
3887     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3888
3889     push @$cmd, '-S' if $conf->{freeze};
3890
3891     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3892
3893     my $guest_agent = parse_guest_agent($conf);
3894
3895     if ($guest_agent->{enabled}) {
3896         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3897         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3898
3899         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3900             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3901             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3902             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3903         } elsif ($guest_agent->{type} eq 'isa') {
3904             push @$devices, '-device', "isa-serial,chardev=qga0";
3905         }
3906     }
3907
3908     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3909     if ($rng && $version_guard->(4, 1, 2)) {
3910         check_rng_source($rng->{source});
3911
3912         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3913         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3914         my $limiter_str = "";
3915         if ($max_bytes) {
3916             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3917         }
3918
3919         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3920         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3921         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3922     }
3923
3924     my $spice_port;
3925
3926     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3927         if ($qxlnum > 1) {
3928             if ($winversion){
3929                 for (my $i = 1; $i < $qxlnum; $i++){
3930                     push @$devices, '-device', print_vga_device(
3931                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3932                 }
3933             } else {
3934                 # assume other OS works like Linux
3935                 my ($ram, $vram) = ("134217728", "67108864");
3936                 if ($vga->{memory}) {
3937                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3938                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3939                 }
3940                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3941                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3942             }
3943         }
3944
3945         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3946
3947         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3948         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3949         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3950
3951         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3952         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3953         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3954
3955         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3956         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3957
3958         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3959         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3960         if ($spice_enhancement->{foldersharing}) {
3961             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3962             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3963         }
3964
3965         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3966         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3967             if $spice_enhancement->{videostreaming};
3968
3969         push @$devices, '-spice', "$spice_opts";
3970     }
3971
3972     # enable balloon by default, unless explicitly disabled
3973     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3974         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3975         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3976         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3977         push @$devices, '-device', $ballooncmd;
3978     }
3979
3980     if ($conf->{watchdog}) {
3981         my $wdopts = parse_watchdog($conf->{watchdog});
3982         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3983         my $watchdog = $wdopts->{model} || 'i6300esb';
3984         push @$devices, '-device', "$watchdog$pciaddr";
3985         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3986     }
3987
3988     my $vollist = [];
3989     my $scsicontroller = {};
3990     my $ahcicontroller = {};
3991     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3992
3993     # Add iscsi initiator name if available
3994     if (my $initiator = get_initiator_name()) {
3995         push @$devices, '-iscsi', "initiator-name=$initiator";
3996     }
3997
3998     PVE::QemuConfig->foreach_volume($conf, sub {
3999         my ($ds, $drive) = @_;
4000
4001         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4002             check_volume_storage_type($storecfg, $drive->{file});
4003             push @$vollist, $drive->{file};
4004         }
4005
4006         # ignore efidisk here, already added in bios/fw handling code above
4007         return if $drive->{interface} eq 'efidisk';
4008         # similar for TPM
4009         return if $drive->{interface} eq 'tpmstate';
4010
4011         $use_virtio = 1 if $ds =~ m/^virtio/;
4012
4013         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4014
4015         if ($drive->{interface} eq 'virtio'){
4016            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4017         }
4018
4019         if ($drive->{interface} eq 'scsi') {
4020
4021             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4022
4023             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4024                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4025
4026             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4027             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4028
4029             my $iothread = '';
4030             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4031                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4032                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4033             } elsif ($drive->{iothread}) {
4034                 log_warn(
4035                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4036                 );
4037             }
4038
4039             my $queues = '';
4040             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4041                 $queues = ",num_queues=$drive->{queues}";
4042             }
4043
4044             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4045                 if !$scsicontroller->{$controller};
4046             $scsicontroller->{$controller}=1;
4047         }
4048
4049         if ($drive->{interface} eq 'sata') {
4050             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4051             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4052             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4053                 if !$ahcicontroller->{$controller};
4054             $ahcicontroller->{$controller}=1;
4055         }
4056
4057         my $pbs_conf = $pbs_backing->{$ds};
4058         my $pbs_name = undef;
4059         if ($pbs_conf) {
4060             $pbs_name = "drive-$ds-pbs";
4061             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4062         }
4063
4064         my $drive_cmd = print_drive_commandline_full(
4065             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4066
4067         # extra protection for templates, but SATA and IDE don't support it..
4068         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4069
4070         push @$devices, '-drive',$drive_cmd;
4071         push @$devices, '-device', print_drivedevice_full(
4072             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4073     });
4074
4075     for (my $i = 0; $i < $MAX_NETS; $i++) {
4076         my $netname = "net$i";
4077
4078         next if !$conf->{$netname};
4079         my $d = parse_net($conf->{$netname});
4080         next if !$d;
4081         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4082
4083         $use_virtio = 1 if $d->{model} eq 'virtio';
4084
4085         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4086
4087         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4088         push @$devices, '-netdev', $netdevfull;
4089
4090         my $netdevicefull = print_netdevice_full(
4091             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4092
4093         push @$devices, '-device', $netdevicefull;
4094     }
4095
4096     if ($conf->{ivshmem}) {
4097         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4098
4099         my $bus;
4100         if ($q35) {
4101             $bus = print_pcie_addr("ivshmem");
4102         } else {
4103             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4104         }
4105
4106         my $ivshmem_name = $ivshmem->{name} // $vmid;
4107         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4108
4109         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4110         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4111             .",size=$ivshmem->{size}M";
4112     }
4113
4114     # pci.4 is nested in pci.1
4115     $bridges->{1} = 1 if $bridges->{4};
4116
4117     if (!$q35) { # add pci bridges
4118         if (min_version($machine_version, 2, 3)) {
4119            $bridges->{1} = 1;
4120            $bridges->{2} = 1;
4121         }
4122         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4123     }
4124
4125     for my $k (sort {$b cmp $a} keys %$bridges) {
4126         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4127
4128         my $k_name = $k;
4129         if ($k == 2 && $legacy_igd) {
4130             $k_name = "$k-igd";
4131         }
4132         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4133         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4134
4135         if ($q35) { # add after -readconfig pve-q35.cfg
4136             splice @$devices, 2, 0, '-device', $devstr;
4137         } else {
4138             unshift @$devices, '-device', $devstr if $k > 0;
4139         }
4140     }
4141
4142     if (!$kvm) {
4143         push @$machineFlags, 'accel=tcg';
4144     }
4145
4146     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4147
4148     my $machine_type_min = $machine_type;
4149     if ($add_pve_version) {
4150         $machine_type_min =~ s/\+pve\d+$//;
4151         $machine_type_min .= "+pve$required_pve_version";
4152     }
4153     push @$machineFlags, "type=${machine_type_min}";
4154
4155     push @$cmd, @$devices;
4156     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4157     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4158     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4159
4160     if (my $vmstate = $conf->{vmstate}) {
4161         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4162         push @$vollist, $vmstate;
4163         push @$cmd, '-loadstate', $statepath;
4164         print "activating and using '$vmstate' as vmstate\n";
4165     }
4166
4167     if (PVE::QemuConfig->is_template($conf)) {
4168         # needed to workaround base volumes being read-only
4169         push @$cmd, '-snapshot';
4170     }
4171
4172     # add custom args
4173     if ($conf->{args}) {
4174         my $aa = PVE::Tools::split_args($conf->{args});
4175         push @$cmd, @$aa;
4176     }
4177
4178     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4179 }
4180
4181 sub check_rng_source {
4182     my ($source) = @_;
4183
4184     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4185     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4186         if ! -e $source;
4187
4188     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4189     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4190         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4191         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4192         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4193             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4194             ." to the host.\n";
4195     }
4196 }
4197
4198 sub spice_port {
4199     my ($vmid) = @_;
4200
4201     my $res = mon_cmd($vmid, 'query-spice');
4202
4203     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4204 }
4205
4206 sub vm_devices_list {
4207     my ($vmid) = @_;
4208
4209     my $res = mon_cmd($vmid, 'query-pci');
4210     my $devices_to_check = [];
4211     my $devices = {};
4212     foreach my $pcibus (@$res) {
4213         push @$devices_to_check, @{$pcibus->{devices}},
4214     }
4215
4216     while (@$devices_to_check) {
4217         my $to_check = [];
4218         for my $d (@$devices_to_check) {
4219             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4220             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4221
4222             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4223             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4224         }
4225         $devices_to_check = $to_check;
4226     }
4227
4228     my $resblock = mon_cmd($vmid, 'query-block');
4229     foreach my $block (@$resblock) {
4230         if($block->{device} =~ m/^drive-(\S+)/){
4231                 $devices->{$1} = 1;
4232         }
4233     }
4234
4235     my $resmice = mon_cmd($vmid, 'query-mice');
4236     foreach my $mice (@$resmice) {
4237         if ($mice->{name} eq 'QEMU HID Tablet') {
4238             $devices->{tablet} = 1;
4239             last;
4240         }
4241     }
4242
4243     # for usb devices there is no query-usb
4244     # but we can iterate over the entries in
4245     # qom-list path=/machine/peripheral
4246     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4247     foreach my $per (@$resperipheral) {
4248         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4249             $devices->{$per->{name}} = 1;
4250         }
4251     }
4252
4253     return $devices;
4254 }
4255
4256 sub vm_deviceplug {
4257     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4258
4259     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4260
4261     my $devices_list = vm_devices_list($vmid);
4262     return 1 if defined($devices_list->{$deviceid});
4263
4264     # add PCI bridge if we need it for the device
4265     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4266
4267     if ($deviceid eq 'tablet') {
4268         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4269     } elsif ($deviceid eq 'keyboard') {
4270         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4271     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4272         my $id = $1;
4273         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4274         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4275     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4276         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4277     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4278         qemu_iothread_add($vmid, $deviceid, $device);
4279
4280         qemu_driveadd($storecfg, $vmid, $device);
4281         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4282
4283         qemu_deviceadd($vmid, $devicefull);
4284         eval { qemu_deviceaddverify($vmid, $deviceid); };
4285         if (my $err = $@) {
4286             eval { qemu_drivedel($vmid, $deviceid); };
4287             warn $@ if $@;
4288             die $err;
4289         }
4290     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4291         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4292         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4293         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4294
4295         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4296
4297         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4298             qemu_iothread_add($vmid, $deviceid, $device);
4299             $devicefull .= ",iothread=iothread-$deviceid";
4300         }
4301
4302         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4303             $devicefull .= ",num_queues=$device->{queues}";
4304         }
4305
4306         qemu_deviceadd($vmid, $devicefull);
4307         qemu_deviceaddverify($vmid, $deviceid);
4308     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4309         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4310         qemu_driveadd($storecfg, $vmid, $device);
4311
4312         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4313         eval { qemu_deviceadd($vmid, $devicefull); };
4314         if (my $err = $@) {
4315             eval { qemu_drivedel($vmid, $deviceid); };
4316             warn $@ if $@;
4317             die $err;
4318         }
4319     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4320         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4321
4322         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4323         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4324         my $use_old_bios_files = undef;
4325         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4326
4327         my $netdevicefull = print_netdevice_full(
4328             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4329         qemu_deviceadd($vmid, $netdevicefull);
4330         eval {
4331             qemu_deviceaddverify($vmid, $deviceid);
4332             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4333         };
4334         if (my $err = $@) {
4335             eval { qemu_netdevdel($vmid, $deviceid); };
4336             warn $@ if $@;
4337             die $err;
4338         }
4339     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4340         my $bridgeid = $2;
4341         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4342         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4343
4344         qemu_deviceadd($vmid, $devicefull);
4345         qemu_deviceaddverify($vmid, $deviceid);
4346     } else {
4347         die "can't hotplug device '$deviceid'\n";
4348     }
4349
4350     return 1;
4351 }
4352
4353 # fixme: this should raise exceptions on error!
4354 sub vm_deviceunplug {
4355     my ($vmid, $conf, $deviceid) = @_;
4356
4357     my $devices_list = vm_devices_list($vmid);
4358     return 1 if !defined($devices_list->{$deviceid});
4359
4360     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4361     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4362
4363     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4364         qemu_devicedel($vmid, $deviceid);
4365     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4366         qemu_devicedel($vmid, $deviceid);
4367         qemu_devicedelverify($vmid, $deviceid);
4368     } elsif ($deviceid =~ m/^usb\d+$/) {
4369         qemu_devicedel($vmid, $deviceid);
4370         qemu_devicedelverify($vmid, $deviceid);
4371     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4372         my $device = parse_drive($deviceid, $conf->{$deviceid});
4373
4374         qemu_devicedel($vmid, $deviceid);
4375         qemu_devicedelverify($vmid, $deviceid);
4376         qemu_drivedel($vmid, $deviceid);
4377         qemu_iothread_del($vmid, $deviceid, $device);
4378     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4379         qemu_devicedel($vmid, $deviceid);
4380         qemu_devicedelverify($vmid, $deviceid);
4381     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4382         my $device = parse_drive($deviceid, $conf->{$deviceid});
4383
4384         qemu_devicedel($vmid, $deviceid);
4385         qemu_devicedelverify($vmid, $deviceid);
4386         qemu_drivedel($vmid, $deviceid);
4387         qemu_deletescsihw($conf, $vmid, $deviceid);
4388
4389         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4390             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4391     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4392         qemu_devicedel($vmid, $deviceid);
4393         qemu_devicedelverify($vmid, $deviceid);
4394         qemu_netdevdel($vmid, $deviceid);
4395     } else {
4396         die "can't unplug device '$deviceid'\n";
4397     }
4398
4399     return 1;
4400 }
4401
4402 sub qemu_spice_usbredir_chardev_add {
4403     my ($vmid, $id) = @_;
4404
4405     mon_cmd($vmid, "chardev-add" , (
4406         id => $id,
4407         backend => {
4408             type => 'spicevmc',
4409             data => {
4410                 type => "usbredir",
4411             },
4412         },
4413     ));
4414 }
4415
4416 sub qemu_deviceadd {
4417     my ($vmid, $devicefull) = @_;
4418
4419     $devicefull = "driver=".$devicefull;
4420     my %options =  split(/[=,]/, $devicefull);
4421
4422     mon_cmd($vmid, "device_add" , %options);
4423 }
4424
4425 sub qemu_devicedel {
4426     my ($vmid, $deviceid) = @_;
4427
4428     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4429 }
4430
4431 sub qemu_iothread_add {
4432     my ($vmid, $deviceid, $device) = @_;
4433
4434     if ($device->{iothread}) {
4435         my $iothreads = vm_iothreads_list($vmid);
4436         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4437     }
4438 }
4439
4440 sub qemu_iothread_del {
4441     my ($vmid, $deviceid, $device) = @_;
4442
4443     if ($device->{iothread}) {
4444         my $iothreads = vm_iothreads_list($vmid);
4445         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4446     }
4447 }
4448
4449 sub qemu_objectadd {
4450     my ($vmid, $objectid, $qomtype) = @_;
4451
4452     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4453
4454     return 1;
4455 }
4456
4457 sub qemu_objectdel {
4458     my ($vmid, $objectid) = @_;
4459
4460     mon_cmd($vmid, "object-del", id => $objectid);
4461
4462     return 1;
4463 }
4464
4465 sub qemu_driveadd {
4466     my ($storecfg, $vmid, $device) = @_;
4467
4468     my $kvmver = get_running_qemu_version($vmid);
4469     my $io_uring = min_version($kvmver, 6, 0);
4470     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4471     $drive =~ s/\\/\\\\/g;
4472     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4473
4474     # If the command succeeds qemu prints: "OK"
4475     return 1 if $ret =~ m/OK/s;
4476
4477     die "adding drive failed: $ret\n";
4478 }
4479
4480 sub qemu_drivedel {
4481     my ($vmid, $deviceid) = @_;
4482
4483     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4484     $ret =~ s/^\s+//;
4485
4486     return 1 if $ret eq "";
4487
4488     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4489     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4490
4491     die "deleting drive $deviceid failed : $ret\n";
4492 }
4493
4494 sub qemu_deviceaddverify {
4495     my ($vmid, $deviceid) = @_;
4496
4497     for (my $i = 0; $i <= 5; $i++) {
4498          my $devices_list = vm_devices_list($vmid);
4499          return 1 if defined($devices_list->{$deviceid});
4500          sleep 1;
4501     }
4502
4503     die "error on hotplug device '$deviceid'\n";
4504 }
4505
4506
4507 sub qemu_devicedelverify {
4508     my ($vmid, $deviceid) = @_;
4509
4510     # need to verify that the device is correctly removed as device_del
4511     # is async and empty return is not reliable
4512
4513     for (my $i = 0; $i <= 5; $i++) {
4514          my $devices_list = vm_devices_list($vmid);
4515          return 1 if !defined($devices_list->{$deviceid});
4516          sleep 1;
4517     }
4518
4519     die "error on hot-unplugging device '$deviceid'\n";
4520 }
4521
4522 sub qemu_findorcreatescsihw {
4523     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4524
4525     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4526
4527     my $scsihwid="$controller_prefix$controller";
4528     my $devices_list = vm_devices_list($vmid);
4529
4530     if (!defined($devices_list->{$scsihwid})) {
4531         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4532     }
4533
4534     return 1;
4535 }
4536
4537 sub qemu_deletescsihw {
4538     my ($conf, $vmid, $opt) = @_;
4539
4540     my $device = parse_drive($opt, $conf->{$opt});
4541
4542     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4543         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4544         return 1;
4545     }
4546
4547     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4548
4549     my $devices_list = vm_devices_list($vmid);
4550     foreach my $opt (keys %{$devices_list}) {
4551         if (is_valid_drivename($opt)) {
4552             my $drive = parse_drive($opt, $conf->{$opt});
4553             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4554                 return 1;
4555             }
4556         }
4557     }
4558
4559     my $scsihwid="scsihw$controller";
4560
4561     vm_deviceunplug($vmid, $conf, $scsihwid);
4562
4563     return 1;
4564 }
4565
4566 sub qemu_add_pci_bridge {
4567     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4568
4569     my $bridges = {};
4570
4571     my $bridgeid;
4572
4573     print_pci_addr($device, $bridges, $arch, $machine_type);
4574
4575     while (my ($k, $v) = each %$bridges) {
4576         $bridgeid = $k;
4577     }
4578     return 1 if !defined($bridgeid) || $bridgeid < 1;
4579
4580     my $bridge = "pci.$bridgeid";
4581     my $devices_list = vm_devices_list($vmid);
4582
4583     if (!defined($devices_list->{$bridge})) {
4584         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4585     }
4586
4587     return 1;
4588 }
4589
4590 sub qemu_set_link_status {
4591     my ($vmid, $device, $up) = @_;
4592
4593     mon_cmd($vmid, "set_link", name => $device,
4594                up => $up ? JSON::true : JSON::false);
4595 }
4596
4597 sub qemu_netdevadd {
4598     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4599
4600     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4601     my %options =  split(/[=,]/, $netdev);
4602
4603     if (defined(my $vhost = $options{vhost})) {
4604         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4605     }
4606
4607     if (defined(my $queues = $options{queues})) {
4608         $options{queues} = $queues + 0;
4609     }
4610
4611     mon_cmd($vmid, "netdev_add",  %options);
4612     return 1;
4613 }
4614
4615 sub qemu_netdevdel {
4616     my ($vmid, $deviceid) = @_;
4617
4618     mon_cmd($vmid, "netdev_del", id => $deviceid);
4619 }
4620
4621 sub qemu_usb_hotplug {
4622     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4623
4624     return if !$device;
4625
4626     # remove the old one first
4627     vm_deviceunplug($vmid, $conf, $deviceid);
4628
4629     # check if xhci controller is necessary and available
4630     my $devicelist = vm_devices_list($vmid);
4631
4632     if (!$devicelist->{xhci}) {
4633         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4634         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4635     }
4636
4637     # print_usbdevice_full expects the parsed device
4638     my $d = parse_usb_device($device->{host});
4639     $d->{usb3} = $device->{usb3};
4640
4641     # add the new one
4642     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4643 }
4644
4645 sub qemu_cpu_hotplug {
4646     my ($vmid, $conf, $vcpus) = @_;
4647
4648     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4649
4650     my $sockets = 1;
4651     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4652     $sockets = $conf->{sockets} if  $conf->{sockets};
4653     my $cores = $conf->{cores} || 1;
4654     my $maxcpus = $sockets * $cores;
4655
4656     $vcpus = $maxcpus if !$vcpus;
4657
4658     die "you can't add more vcpus than maxcpus\n"
4659         if $vcpus > $maxcpus;
4660
4661     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4662
4663     if ($vcpus < $currentvcpus) {
4664
4665         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4666
4667             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4668                 qemu_devicedel($vmid, "cpu$i");
4669                 my $retry = 0;
4670                 my $currentrunningvcpus = undef;
4671                 while (1) {
4672                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4673                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4674                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4675                     $retry++;
4676                     sleep 1;
4677                 }
4678                 #update conf after each succesfull cpu unplug
4679                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4680                 PVE::QemuConfig->write_config($vmid, $conf);
4681             }
4682         } else {
4683             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4684         }
4685
4686         return;
4687     }
4688
4689     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4690     die "vcpus in running vm does not match its configuration\n"
4691         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4692
4693     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4694
4695         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4696             my $cpustr = print_cpu_device($conf, $i);
4697             qemu_deviceadd($vmid, $cpustr);
4698
4699             my $retry = 0;
4700             my $currentrunningvcpus = undef;
4701             while (1) {
4702                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4703                 last if scalar(@{$currentrunningvcpus}) == $i;
4704                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4705                 sleep 1;
4706                 $retry++;
4707             }
4708             #update conf after each succesfull cpu hotplug
4709             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4710             PVE::QemuConfig->write_config($vmid, $conf);
4711         }
4712     } else {
4713
4714         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4715             mon_cmd($vmid, "cpu-add", id => int($i));
4716         }
4717     }
4718 }
4719
4720 sub qemu_block_set_io_throttle {
4721     my ($vmid, $deviceid,
4722         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4723         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4724         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4725         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4726
4727     return if !check_running($vmid) ;
4728
4729     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4730         bps => int($bps),
4731         bps_rd => int($bps_rd),
4732         bps_wr => int($bps_wr),
4733         iops => int($iops),
4734         iops_rd => int($iops_rd),
4735         iops_wr => int($iops_wr),
4736         bps_max => int($bps_max),
4737         bps_rd_max => int($bps_rd_max),
4738         bps_wr_max => int($bps_wr_max),
4739         iops_max => int($iops_max),
4740         iops_rd_max => int($iops_rd_max),
4741         iops_wr_max => int($iops_wr_max),
4742         bps_max_length => int($bps_max_length),
4743         bps_rd_max_length => int($bps_rd_max_length),
4744         bps_wr_max_length => int($bps_wr_max_length),
4745         iops_max_length => int($iops_max_length),
4746         iops_rd_max_length => int($iops_rd_max_length),
4747         iops_wr_max_length => int($iops_wr_max_length),
4748     );
4749
4750 }
4751
4752 sub qemu_block_resize {
4753     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4754
4755     my $running = check_running($vmid);
4756
4757     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4758
4759     return if !$running;
4760
4761     my $padding = (1024 - $size % 1024) % 1024;
4762     $size = $size + $padding;
4763
4764     mon_cmd(
4765         $vmid,
4766         "block_resize",
4767         device => $deviceid,
4768         size => int($size),
4769         timeout => 60,
4770     );
4771 }
4772
4773 sub qemu_volume_snapshot {
4774     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4775
4776     my $running = check_running($vmid);
4777
4778     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4779         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4780     } else {
4781         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4782     }
4783 }
4784
4785 sub qemu_volume_snapshot_delete {
4786     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4787
4788     my $running = check_running($vmid);
4789
4790     if($running) {
4791
4792         $running = undef;
4793         my $conf = PVE::QemuConfig->load_config($vmid);
4794         PVE::QemuConfig->foreach_volume($conf, sub {
4795             my ($ds, $drive) = @_;
4796             $running = 1 if $drive->{file} eq $volid;
4797         });
4798     }
4799
4800     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4801         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4802     } else {
4803         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4804     }
4805 }
4806
4807 sub set_migration_caps {
4808     my ($vmid, $savevm) = @_;
4809
4810     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4811
4812     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4813     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4814
4815     my $cap_ref = [];
4816
4817     my $enabled_cap = {
4818         "auto-converge" => 1,
4819         "xbzrle" => 1,
4820         "x-rdma-pin-all" => 0,
4821         "zero-blocks" => 0,
4822         "compress" => 0,
4823         "dirty-bitmaps" => $dirty_bitmaps,
4824     };
4825
4826     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4827
4828     for my $supported_capability (@$supported_capabilities) {
4829         push @$cap_ref, {
4830             capability => $supported_capability->{capability},
4831             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4832         };
4833     }
4834
4835     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4836 }
4837
4838 sub foreach_volid {
4839     my ($conf, $func, @param) = @_;
4840
4841     my $volhash = {};
4842
4843     my $test_volid = sub {
4844         my ($key, $drive, $snapname) = @_;
4845
4846         my $volid = $drive->{file};
4847         return if !$volid;
4848
4849         $volhash->{$volid}->{cdrom} //= 1;
4850         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4851
4852         my $replicate = $drive->{replicate} // 1;
4853         $volhash->{$volid}->{replicate} //= 0;
4854         $volhash->{$volid}->{replicate} = 1 if $replicate;
4855
4856         $volhash->{$volid}->{shared} //= 0;
4857         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4858
4859         $volhash->{$volid}->{referenced_in_config} //= 0;
4860         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4861
4862         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4863             if defined($snapname);
4864
4865         my $size = $drive->{size};
4866         $volhash->{$volid}->{size} //= $size if $size;
4867
4868         $volhash->{$volid}->{is_vmstate} //= 0;
4869         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4870
4871         $volhash->{$volid}->{is_tpmstate} //= 0;
4872         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4873
4874         $volhash->{$volid}->{is_unused} //= 0;
4875         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4876
4877         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4878     };
4879
4880     my $include_opts = {
4881         extra_keys => ['vmstate'],
4882         include_unused => 1,
4883     };
4884
4885     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4886     foreach my $snapname (keys %{$conf->{snapshots}}) {
4887         my $snap = $conf->{snapshots}->{$snapname};
4888         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4889     }
4890
4891     foreach my $volid (keys %$volhash) {
4892         &$func($volid, $volhash->{$volid}, @param);
4893     }
4894 }
4895
4896 my $fast_plug_option = {
4897     'lock' => 1,
4898     'name' => 1,
4899     'onboot' => 1,
4900     'shares' => 1,
4901     'startup' => 1,
4902     'description' => 1,
4903     'protection' => 1,
4904     'vmstatestorage' => 1,
4905     'hookscript' => 1,
4906     'tags' => 1,
4907 };
4908
4909 for my $opt (keys %$confdesc_cloudinit) {
4910     $fast_plug_option->{$opt} = 1;
4911 };
4912
4913 # hotplug changes in [PENDING]
4914 # $selection hash can be used to only apply specified options, for
4915 # example: { cores => 1 } (only apply changed 'cores')
4916 # $errors ref is used to return error messages
4917 sub vmconfig_hotplug_pending {
4918     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4919
4920     my $defaults = load_defaults();
4921     my $arch = get_vm_arch($conf);
4922     my $machine_type = get_vm_machine($conf, undef, $arch);
4923
4924     # commit values which do not have any impact on running VM first
4925     # Note: those option cannot raise errors, we we do not care about
4926     # $selection and always apply them.
4927
4928     my $add_error = sub {
4929         my ($opt, $msg) = @_;
4930         $errors->{$opt} = "hotplug problem - $msg";
4931     };
4932
4933     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4934
4935     my $cloudinit_record_changed = sub {
4936         my ($conf, $opt, $old, $new) = @_;
4937         return if !$cloudinit_pending_properties->{$opt};
4938
4939         my $ci = ($conf->{cloudinit} //= {});
4940
4941         my $recorded = $ci->{$opt};
4942         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4943
4944         if (defined($new)) {
4945             if (defined($old)) {
4946                 # an existing value is being modified
4947                 if (defined($recorded)) {
4948                     # the value was already not in sync
4949                     if ($new eq $recorded) {
4950                         # a value is being reverted to the cloud-init state:
4951                         delete $ci->{$opt};
4952                         delete $added{$opt};
4953                     } else {
4954                         # the value was changed multiple times, do nothing
4955                     }
4956                 } elsif ($added{$opt}) {
4957                     # the value had been marked as added and is being changed, do nothing
4958                 } else {
4959                     # the value is new, record it:
4960                     $ci->{$opt} = $old;
4961                 }
4962             } else {
4963                 # a new value is being added
4964                 if (defined($recorded)) {
4965                     # it was already not in sync
4966                     if ($new eq $recorded) {
4967                         # a value is being reverted to the cloud-init state:
4968                         delete $ci->{$opt};
4969                         delete $added{$opt};
4970                     } else {
4971                         # the value had temporarily been removed, do nothing
4972                     }
4973                 } elsif ($added{$opt}) {
4974                     # the value had been marked as added already, do nothing
4975                 } else {
4976                     # the value is new, add it
4977                     $added{$opt} = 1;
4978                 }
4979             }
4980         } elsif (!defined($old)) {
4981             # a non-existent value is being removed? ignore...
4982         } else {
4983             # a value is being deleted
4984             if (defined($recorded)) {
4985                 # a value was already recorded, just keep it
4986             } elsif ($added{$opt}) {
4987                 # the value was marked as added, remove it
4988                 delete $added{$opt};
4989             } else {
4990                 # a previously unrecorded value is being removed, record the old value:
4991                 $ci->{$opt} = $old;
4992             }
4993         }
4994
4995         my $added = join(',', sort keys %added);
4996         $ci->{added} = $added if length($added);
4997     };
4998
4999     my $changes = 0;
5000     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5001         if ($fast_plug_option->{$opt}) {
5002             my $new = delete $conf->{pending}->{$opt};
5003             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
5004             $conf->{$opt} = $new;
5005             $changes = 1;
5006         }
5007     }
5008
5009     if ($changes) {
5010         PVE::QemuConfig->write_config($vmid, $conf);
5011     }
5012
5013     my $ostype = $conf->{ostype};
5014     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5015     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5016     my $usb_hotplug = $hotplug_features->{usb}
5017         && min_version($version, 7, 1)
5018         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5019
5020     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5021     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5022
5023     foreach my $opt (sort keys %$pending_delete_hash) {
5024         next if $selection && !$selection->{$opt};
5025         my $force = $pending_delete_hash->{$opt}->{force};
5026         eval {
5027             if ($opt eq 'hotplug') {
5028                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5029             } elsif ($opt eq 'tablet') {
5030                 die "skip\n" if !$hotplug_features->{usb};
5031                 if ($defaults->{tablet}) {
5032                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5033                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5034                         if $arch eq 'aarch64';
5035                 } else {
5036                     vm_deviceunplug($vmid, $conf, 'tablet');
5037                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5038                 }
5039             } elsif ($opt =~ m/^usb(\d+)$/) {
5040                 my $index = $1;
5041                 die "skip\n" if !$usb_hotplug;
5042                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5043                 vm_deviceunplug($vmid, $conf, $opt);
5044             } elsif ($opt eq 'vcpus') {
5045                 die "skip\n" if !$hotplug_features->{cpu};
5046                 qemu_cpu_hotplug($vmid, $conf, undef);
5047             } elsif ($opt eq 'balloon') {
5048                 # enable balloon device is not hotpluggable
5049                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5050                 # here we reset the ballooning value to memory
5051                 my $balloon = $conf->{memory} || $defaults->{memory};
5052                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5053             } elsif ($fast_plug_option->{$opt}) {
5054                 # do nothing
5055             } elsif ($opt =~ m/^net(\d+)$/) {
5056                 die "skip\n" if !$hotplug_features->{network};
5057                 vm_deviceunplug($vmid, $conf, $opt);
5058             } elsif (is_valid_drivename($opt)) {
5059                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5060                 vm_deviceunplug($vmid, $conf, $opt);
5061                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5062             } elsif ($opt =~ m/^memory$/) {
5063                 die "skip\n" if !$hotplug_features->{memory};
5064                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5065             } elsif ($opt eq 'cpuunits') {
5066                 $cgroup->change_cpu_shares(undef);
5067             } elsif ($opt eq 'cpulimit') {
5068                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5069             } else {
5070                 die "skip\n";
5071             }
5072         };
5073         if (my $err = $@) {
5074             &$add_error($opt, $err) if $err ne "skip\n";
5075         } else {
5076             my $old = delete $conf->{$opt};
5077             $cloudinit_record_changed->($conf, $opt, $old, undef);
5078             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5079         }
5080     }
5081
5082     my $cloudinit_opt;
5083     foreach my $opt (keys %{$conf->{pending}}) {
5084         next if $selection && !$selection->{$opt};
5085         my $value = $conf->{pending}->{$opt};
5086         eval {
5087             if ($opt eq 'hotplug') {
5088                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5089             } elsif ($opt eq 'tablet') {
5090                 die "skip\n" if !$hotplug_features->{usb};
5091                 if ($value == 1) {
5092                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5093                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5094                         if $arch eq 'aarch64';
5095                 } elsif ($value == 0) {
5096                     vm_deviceunplug($vmid, $conf, 'tablet');
5097                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5098                 }
5099             } elsif ($opt =~ m/^usb(\d+)$/) {
5100                 my $index = $1;
5101                 die "skip\n" if !$usb_hotplug;
5102                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5103                 my $id = $opt;
5104                 if ($d->{host} eq 'spice')  {
5105                     $id = "usbredirdev$index";
5106                 }
5107                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5108             } elsif ($opt eq 'vcpus') {
5109                 die "skip\n" if !$hotplug_features->{cpu};
5110                 qemu_cpu_hotplug($vmid, $conf, $value);
5111             } elsif ($opt eq 'balloon') {
5112                 # enable/disable balloning device is not hotpluggable
5113                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5114                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5115                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5116
5117                 # allow manual ballooning if shares is set to zero
5118                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5119                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5120                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5121                 }
5122             } elsif ($opt =~ m/^net(\d+)$/) {
5123                 # some changes can be done without hotplug
5124                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5125                                     $vmid, $opt, $value, $arch, $machine_type);
5126             } elsif (is_valid_drivename($opt)) {
5127                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5128                 # some changes can be done without hotplug
5129                 my $drive = parse_drive($opt, $value);
5130                 if (drive_is_cloudinit($drive)) {
5131                     $cloudinit_opt = [$opt, $drive];
5132                     # apply all the other changes first, then generate the cloudinit disk
5133                     die "skip\n";
5134                 }
5135                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5136                                      $vmid, $opt, $value, $arch, $machine_type);
5137             } elsif ($opt =~ m/^memory$/) { #dimms
5138                 die "skip\n" if !$hotplug_features->{memory};
5139                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5140             } elsif ($opt eq 'cpuunits') {
5141                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5142                 $cgroup->change_cpu_shares($new_cpuunits);
5143             } elsif ($opt eq 'cpulimit') {
5144                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5145                 $cgroup->change_cpu_quota($cpulimit, 100000);
5146             } elsif ($opt eq 'agent') {
5147                 vmconfig_update_agent($conf, $opt, $value);
5148             } else {
5149                 die "skip\n";  # skip non-hot-pluggable options
5150             }
5151         };
5152         if (my $err = $@) {
5153             &$add_error($opt, $err) if $err ne "skip\n";
5154         } else {
5155             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5156             $conf->{$opt} = $value;
5157             delete $conf->{pending}->{$opt};
5158         }
5159     }
5160
5161     if (defined($cloudinit_opt)) {
5162         my ($opt, $drive) = @$cloudinit_opt;
5163         my $value = $conf->{pending}->{$opt};
5164         eval {
5165             my $temp = {%$conf, $opt => $value};
5166             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5167             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5168                                  $vmid, $opt, $value, $arch, $machine_type);
5169         };
5170         if (my $err = $@) {
5171             &$add_error($opt, $err) if $err ne "skip\n";
5172         } else {
5173             $conf->{$opt} = $value;
5174             delete $conf->{pending}->{$opt};
5175         }
5176     }
5177
5178     # unplug xhci controller if no usb device is left
5179     if ($usb_hotplug) {
5180         my $has_usb = 0;
5181         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5182             next if !defined($conf->{"usb$i"});
5183             $has_usb = 1;
5184             last;
5185         }
5186         if (!$has_usb) {
5187             vm_deviceunplug($vmid, $conf, 'xhci');
5188         }
5189     }
5190
5191     PVE::QemuConfig->write_config($vmid, $conf);
5192
5193     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5194         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5195     }
5196 }
5197
5198 sub try_deallocate_drive {
5199     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5200
5201     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5202         my $volid = $drive->{file};
5203         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5204             my $sid = PVE::Storage::parse_volume_id($volid);
5205             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5206
5207             # check if the disk is really unused
5208             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5209                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5210             PVE::Storage::vdisk_free($storecfg, $volid);
5211             return 1;
5212         } else {
5213             # If vm is not owner of this disk remove from config
5214             return 1;
5215         }
5216     }
5217
5218     return;
5219 }
5220
5221 sub vmconfig_delete_or_detach_drive {
5222     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5223
5224     my $drive = parse_drive($opt, $conf->{$opt});
5225
5226     my $rpcenv = PVE::RPCEnvironment::get();
5227     my $authuser = $rpcenv->get_user();
5228
5229     if ($force) {
5230         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5231         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5232     } else {
5233         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5234     }
5235 }
5236
5237
5238
5239 sub vmconfig_apply_pending {
5240     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5241
5242     return if !scalar(keys %{$conf->{pending}});
5243
5244     my $add_apply_error = sub {
5245         my ($opt, $msg) = @_;
5246         my $err_msg = "unable to apply pending change $opt : $msg";
5247         $errors->{$opt} = $err_msg;
5248         warn $err_msg;
5249     };
5250
5251     # cold plug
5252
5253     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5254     foreach my $opt (sort keys %$pending_delete_hash) {
5255         my $force = $pending_delete_hash->{$opt}->{force};
5256         eval {
5257             if ($opt =~ m/^unused/) {
5258                 die "internal error";
5259             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5260                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5261             }
5262         };
5263         if (my $err = $@) {
5264             $add_apply_error->($opt, $err);
5265         } else {
5266             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5267             delete $conf->{$opt};
5268         }
5269     }
5270
5271     PVE::QemuConfig->cleanup_pending($conf);
5272
5273     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5274
5275     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5276         next if $opt eq 'delete'; # just to be sure
5277         eval {
5278             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5279                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5280             }
5281         };
5282         if (my $err = $@) {
5283             $add_apply_error->($opt, $err);
5284         } else {
5285
5286             if (is_valid_drivename($opt)) {
5287                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5288                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5289             }
5290
5291             $conf->{$opt} = delete $conf->{pending}->{$opt};
5292         }
5293     }
5294
5295     # write all changes at once to avoid unnecessary i/o
5296     PVE::QemuConfig->write_config($vmid, $conf);
5297     if ($generate_cloudinit) {
5298         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5299             # After successful generation and if there were changes to be applied, update the
5300             # config to drop the {cloudinit} entry.
5301             PVE::QemuConfig->write_config($vmid, $conf);
5302         }
5303     }
5304 }
5305
5306 sub vmconfig_update_net {
5307     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5308
5309     my $newnet = parse_net($value);
5310
5311     if ($conf->{$opt}) {
5312         my $oldnet = parse_net($conf->{$opt});
5313
5314         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5315             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5316             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5317             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5318
5319             # for non online change, we try to hot-unplug
5320             die "skip\n" if !$hotplug;
5321             vm_deviceunplug($vmid, $conf, $opt);
5322         } else {
5323
5324             die "internal error" if $opt !~ m/net(\d+)/;
5325             my $iface = "tap${vmid}i$1";
5326
5327             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5328                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5329                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5330                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5331                 PVE::Network::tap_unplug($iface);
5332
5333                 if ($have_sdn) {
5334                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5335                 } else {
5336                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5337                 }
5338             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5339                 # Rate can be applied on its own but any change above needs to
5340                 # include the rate in tap_plug since OVS resets everything.
5341                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5342             }
5343
5344             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5345                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5346             }
5347
5348             return 1;
5349         }
5350     }
5351
5352     if ($hotplug) {
5353         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5354     } else {
5355         die "skip\n";
5356     }
5357 }
5358
5359 sub vmconfig_update_agent {
5360     my ($conf, $opt, $value) = @_;
5361
5362     die "skip\n" if !$conf->{$opt};
5363
5364     my $hotplug_options = { fstrim_cloned_disks => 1 };
5365
5366     my $old_agent = parse_guest_agent($conf);
5367     my $agent = parse_guest_agent({$opt => $value});
5368
5369     for my $option (keys %$agent) { # added/changed options
5370         next if defined($hotplug_options->{$option});
5371         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5372     }
5373
5374     for my $option (keys %$old_agent) { # removed options
5375         next if defined($hotplug_options->{$option});
5376         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5377     }
5378
5379     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5380 }
5381
5382 sub vmconfig_update_disk {
5383     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5384
5385     my $drive = parse_drive($opt, $value);
5386
5387     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5388         my $media = $drive->{media} || 'disk';
5389         my $oldmedia = $old_drive->{media} || 'disk';
5390         die "unable to change media type\n" if $media ne $oldmedia;
5391
5392         if (!drive_is_cdrom($old_drive)) {
5393
5394             if ($drive->{file} ne $old_drive->{file}) {
5395
5396                 die "skip\n" if !$hotplug;
5397
5398                 # unplug and register as unused
5399                 vm_deviceunplug($vmid, $conf, $opt);
5400                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5401
5402             } else {
5403                 # update existing disk
5404
5405                 # skip non hotpluggable value
5406                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5407                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5408                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5409                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5410                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5411                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5412                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5413                     die "skip\n";
5414                 }
5415
5416                 # apply throttle
5417                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5418                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5419                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5420                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5421                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5422                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5423                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5424                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5425                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5426                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5427                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5428                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5429                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5430                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5431                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5432                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5433                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5434                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5435
5436                     qemu_block_set_io_throttle(
5437                         $vmid,"drive-$opt",
5438                         ($drive->{mbps} || 0)*1024*1024,
5439                         ($drive->{mbps_rd} || 0)*1024*1024,
5440                         ($drive->{mbps_wr} || 0)*1024*1024,
5441                          $drive->{iops} || 0,
5442                          $drive->{iops_rd} || 0,
5443                          $drive->{iops_wr} || 0,
5444                         ($drive->{mbps_max} || 0)*1024*1024,
5445                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5446                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5447                          $drive->{iops_max} || 0,
5448                          $drive->{iops_rd_max} || 0,
5449                          $drive->{iops_wr_max} || 0,
5450                          $drive->{bps_max_length} || 1,
5451                          $drive->{bps_rd_max_length} || 1,
5452                          $drive->{bps_wr_max_length} || 1,
5453                          $drive->{iops_max_length} || 1,
5454                          $drive->{iops_rd_max_length} || 1,
5455                          $drive->{iops_wr_max_length} || 1,
5456                     );
5457
5458                 }
5459
5460                 return 1;
5461             }
5462
5463         } else { # cdrom
5464
5465             if ($drive->{file} eq 'none') {
5466                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5467                 if (drive_is_cloudinit($old_drive)) {
5468                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5469                 }
5470             } else {
5471                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5472
5473                 # force eject if locked
5474                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5475
5476                 if ($path) {
5477                     mon_cmd($vmid, "blockdev-change-medium",
5478                         id => "$opt", filename => "$path");
5479                 }
5480             }
5481
5482             return 1;
5483         }
5484     }
5485
5486     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5487     # hotplug new disks
5488     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5489     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5490 }
5491
5492 sub vmconfig_update_cloudinit_drive {
5493     my ($storecfg, $conf, $vmid) = @_;
5494
5495     my $cloudinit_ds = undef;
5496     my $cloudinit_drive = undef;
5497
5498     PVE::QemuConfig->foreach_volume($conf, sub {
5499         my ($ds, $drive) = @_;
5500         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5501             $cloudinit_ds = $ds;
5502             $cloudinit_drive = $drive;
5503         }
5504     });
5505
5506     return if !$cloudinit_drive;
5507
5508     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5509         PVE::QemuConfig->write_config($vmid, $conf);
5510     }
5511
5512     my $running = PVE::QemuServer::check_running($vmid);
5513
5514     if ($running) {
5515         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5516         if ($path) {
5517             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5518             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5519         }
5520     }
5521 }
5522
5523 # called in locked context by incoming migration
5524 sub vm_migrate_get_nbd_disks {
5525     my ($storecfg, $conf, $replicated_volumes) = @_;
5526
5527     my $local_volumes = {};
5528     PVE::QemuConfig->foreach_volume($conf, sub {
5529         my ($ds, $drive) = @_;
5530
5531         return if drive_is_cdrom($drive);
5532         return if $ds eq 'tpmstate0';
5533
5534         my $volid = $drive->{file};
5535
5536         return if !$volid;
5537
5538         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5539
5540         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5541         return if $scfg->{shared};
5542
5543         # replicated disks re-use existing state via bitmap
5544         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5545         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5546     });
5547     return $local_volumes;
5548 }
5549
5550 # called in locked context by incoming migration
5551 sub vm_migrate_alloc_nbd_disks {
5552     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5553
5554     my $nbd = {};
5555     foreach my $opt (sort keys %$source_volumes) {
5556         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5557
5558         if ($use_existing) {
5559             $nbd->{$opt}->{drivestr} = print_drive($drive);
5560             $nbd->{$opt}->{volid} = $volid;
5561             $nbd->{$opt}->{replicated} = 1;
5562             next;
5563         }
5564
5565         # storage mapping + volname = regular migration
5566         # storage mapping + format = remote migration
5567         # order of precedence, filtered by whether storage supports it:
5568         # 1. explicit requested format
5569         # 2. format of current volume
5570         # 3. default format of storage
5571         if (!$storagemap->{identity}) {
5572             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5573             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5574             if (!$format || !grep { $format eq $_ } @$validFormats) {
5575                 if ($volname) {
5576                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5577                     my $fileFormat = qemu_img_format($scfg, $volname);
5578                     $format = $fileFormat
5579                         if grep { $fileFormat eq $_ } @$validFormats;
5580                 }
5581                 $format //= $defFormat;
5582             }
5583         } else {
5584             # can't happen for remote migration, so $volname is always defined
5585             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5586             $format = qemu_img_format($scfg, $volname);
5587         }
5588
5589         my $size = $drive->{size} / 1024;
5590         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5591         my $newdrive = $drive;
5592         $newdrive->{format} = $format;
5593         $newdrive->{file} = $newvolid;
5594         my $drivestr = print_drive($newdrive);
5595         $nbd->{$opt}->{drivestr} = $drivestr;
5596         $nbd->{$opt}->{volid} = $newvolid;
5597     }
5598
5599     return $nbd;
5600 }
5601
5602 # see vm_start_nolock for parameters, additionally:
5603 # migrate_opts:
5604 #   storagemap = parsed storage map for allocating NBD disks
5605 sub vm_start {
5606     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5607
5608     return PVE::QemuConfig->lock_config($vmid, sub {
5609         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5610
5611         die "you can't start a vm if it's a template\n"
5612             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5613
5614         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5615         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5616
5617         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5618
5619         if ($has_backup_lock && $running) {
5620             # a backup is currently running, attempt to start the guest in the
5621             # existing QEMU instance
5622             return vm_resume($vmid);
5623         }
5624
5625         PVE::QemuConfig->check_lock($conf)
5626             if !($params->{skiplock} || $has_suspended_lock);
5627
5628         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5629
5630         die "VM $vmid already running\n" if $running;
5631
5632         if (my $storagemap = $migrate_opts->{storagemap}) {
5633             my $replicated = $migrate_opts->{replicated_volumes};
5634             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5635             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5636
5637             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5638                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5639             }
5640         }
5641
5642         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5643     });
5644 }
5645
5646
5647 # params:
5648 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5649 #   skiplock => 0/1, skip checking for config lock
5650 #   skiptemplate => 0/1, skip checking whether VM is template
5651 #   forcemachine => to force QEMU machine (rollback/migration)
5652 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5653 #   timeout => in seconds
5654 #   paused => start VM in paused state (backup)
5655 #   resume => resume from hibernation
5656 #   pbs-backing => {
5657 #      sata0 => {
5658 #         repository
5659 #         snapshot
5660 #         keyfile
5661 #         archive
5662 #      },
5663 #      virtio2 => ...
5664 #   }
5665 # migrate_opts:
5666 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5667 #   migratedfrom => source node
5668 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5669 #   network => CIDR of migration network
5670 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5671 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5672 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5673 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5674 #       contained in config
5675 sub vm_start_nolock {
5676     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5677
5678     my $statefile = $params->{statefile};
5679     my $resume = $params->{resume};
5680
5681     my $migratedfrom = $migrate_opts->{migratedfrom};
5682     my $migration_type = $migrate_opts->{type};
5683
5684     my $res = {};
5685
5686     # clean up leftover reboot request files
5687     eval { clear_reboot_request($vmid); };
5688     warn $@ if $@;
5689
5690     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5691         vmconfig_apply_pending($vmid, $conf, $storecfg);
5692         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5693     }
5694
5695     # don't regenerate the ISO if the VM is started as part of a live migration
5696     # this way we can reuse the old ISO with the correct config
5697     if (!$migratedfrom) {
5698         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5699             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5700             # $conf->{cloudinit}, so we could just not do this?
5701             # But we do it above, so for now let's be consistent.
5702             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5703         }
5704     }
5705
5706     # override offline migrated volumes, conf is out of date still
5707     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5708         for my $key (sort keys $offline_volumes->%*) {
5709             my $parsed = parse_drive($key, $conf->{$key});
5710             $parsed->{file} = $offline_volumes->{$key};
5711             $conf->{$key} = print_drive($parsed);
5712         }
5713     }
5714
5715     my $defaults = load_defaults();
5716
5717     # set environment variable useful inside network script
5718     # for remote migration the config is available on the target node!
5719     if (!$migrate_opts->{remote_node}) {
5720         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5721     }
5722
5723     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5724
5725     my $forcemachine = $params->{forcemachine};
5726     my $forcecpu = $params->{forcecpu};
5727     if ($resume) {
5728         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5729         $forcemachine = $conf->{runningmachine};
5730         $forcecpu = $conf->{runningcpu};
5731         print "Resuming suspended VM\n";
5732     }
5733
5734     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5735         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5736
5737     my $migration_ip;
5738     my $get_migration_ip = sub {
5739         my ($nodename) = @_;
5740
5741         return $migration_ip if defined($migration_ip);
5742
5743         my $cidr = $migrate_opts->{network};
5744
5745         if (!defined($cidr)) {
5746             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5747             $cidr = $dc_conf->{migration}->{network};
5748         }
5749
5750         if (defined($cidr)) {
5751             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5752
5753             die "could not get IP: no address configured on local " .
5754                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5755
5756             die "could not get IP: multiple addresses configured on local " .
5757                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5758
5759             $migration_ip = @$ips[0];
5760         }
5761
5762         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5763             if !defined($migration_ip);
5764
5765         return $migration_ip;
5766     };
5767
5768     if ($statefile) {
5769         if ($statefile eq 'tcp') {
5770             my $migrate = $res->{migrate} = { proto => 'tcp' };
5771             $migrate->{addr} = "localhost";
5772             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5773             my $nodename = nodename();
5774
5775             if (!defined($migration_type)) {
5776                 if (defined($datacenterconf->{migration}->{type})) {
5777                     $migration_type = $datacenterconf->{migration}->{type};
5778                 } else {
5779                     $migration_type = 'secure';
5780                 }
5781             }
5782
5783             if ($migration_type eq 'insecure') {
5784                 $migrate->{addr} = $get_migration_ip->($nodename);
5785                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5786             }
5787
5788             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5789             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5790             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5791             push @$cmd, '-incoming', $migrate->{uri};
5792             push @$cmd, '-S';
5793
5794         } elsif ($statefile eq 'unix') {
5795             # should be default for secure migrations as a ssh TCP forward
5796             # tunnel is not deterministic reliable ready and fails regurarly
5797             # to set up in time, so use UNIX socket forwards
5798             my $migrate = $res->{migrate} = { proto => 'unix' };
5799             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5800             unlink $migrate->{addr};
5801
5802             $migrate->{uri} = "unix:$migrate->{addr}";
5803             push @$cmd, '-incoming', $migrate->{uri};
5804             push @$cmd, '-S';
5805
5806         } elsif (-e $statefile) {
5807             push @$cmd, '-loadstate', $statefile;
5808         } else {
5809             my $statepath = PVE::Storage::path($storecfg, $statefile);
5810             push @$vollist, $statefile;
5811             push @$cmd, '-loadstate', $statepath;
5812         }
5813     } elsif ($params->{paused}) {
5814         push @$cmd, '-S';
5815     }
5816
5817     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5818
5819     my $pci_devices = {}; # host pci devices
5820     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5821         my $dev = $conf->{"hostpci$i"} or next;
5822         $pci_devices->{$i} = parse_hostpci($dev);
5823     }
5824
5825     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5826     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5827
5828     # map to a flat list of pci ids
5829     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5830
5831     # reserve all PCI IDs before actually doing anything with them
5832     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5833
5834     eval {
5835         my $uuid;
5836         for my $id (sort keys %$pci_devices) {
5837             my $d = $pci_devices->{$id};
5838             for my $dev ($d->{pciid}->@*) {
5839                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5840
5841                 # nvidia grid needs the uuid of the mdev as qemu parameter
5842                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5843                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5844                 }
5845             }
5846         }
5847         push @$cmd, '-uuid', $uuid if defined($uuid);
5848     };
5849     if (my $err = $@) {
5850         eval { cleanup_pci_devices($vmid, $conf) };
5851         warn $@ if $@;
5852         die $err;
5853     }
5854
5855     PVE::Storage::activate_volumes($storecfg, $vollist);
5856
5857     eval {
5858         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5859     };
5860     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5861     # timeout should be more than enough here...
5862     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5863
5864     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5865
5866     my %run_params = (
5867         timeout => $statefile ? undef : $start_timeout,
5868         umask => 0077,
5869         noerr => 1,
5870     );
5871
5872     # when migrating, prefix QEMU output so other side can pick up any
5873     # errors that might occur and show the user
5874     if ($migratedfrom) {
5875         $run_params{quiet} = 1;
5876         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5877     }
5878
5879     my %systemd_properties = (
5880         Slice => 'qemu.slice',
5881         KillMode => 'process',
5882         SendSIGKILL => 0,
5883         TimeoutStopUSec => ULONG_MAX, # infinity
5884     );
5885
5886     if (PVE::CGroup::cgroup_mode() == 2) {
5887         $systemd_properties{CPUWeight} = $cpuunits;
5888     } else {
5889         $systemd_properties{CPUShares} = $cpuunits;
5890     }
5891
5892     if (my $cpulimit = $conf->{cpulimit}) {
5893         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5894     }
5895     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5896
5897     my $run_qemu = sub {
5898         PVE::Tools::run_fork sub {
5899             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5900
5901             my $tpmpid;
5902             if (my $tpm = $conf->{tpmstate0}) {
5903                 # start the TPM emulator so QEMU can connect on start
5904                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5905             }
5906
5907             my $exitcode = run_command($cmd, %run_params);
5908             if ($exitcode) {
5909                 if ($tpmpid) {
5910                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5911                     kill 'TERM', $tpmpid;
5912                 }
5913                 die "QEMU exited with code $exitcode\n";
5914             }
5915         };
5916     };
5917
5918     if ($conf->{hugepages}) {
5919
5920         my $code = sub {
5921             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5922             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5923
5924             PVE::QemuServer::Memory::hugepages_mount();
5925             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5926
5927             eval { $run_qemu->() };
5928             if (my $err = $@) {
5929                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5930                     if !$conf->{keephugepages};
5931                 die $err;
5932             }
5933
5934             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5935                 if !$conf->{keephugepages};
5936         };
5937         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5938
5939     } else {
5940         eval { $run_qemu->() };
5941     }
5942
5943     if (my $err = $@) {
5944         # deactivate volumes if start fails
5945         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5946         warn $@ if $@;
5947         eval { cleanup_pci_devices($vmid, $conf) };
5948         warn $@ if $@;
5949
5950         die "start failed: $err";
5951     }
5952
5953     # re-reserve all PCI IDs now that we can know the actual VM PID
5954     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5955     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5956     warn $@ if $@;
5957
5958     if (defined($res->{migrate})) {
5959         print "migration listens on $res->{migrate}->{uri}\n";
5960     } elsif ($statefile) {
5961         eval { mon_cmd($vmid, "cont"); };
5962         warn $@ if $@;
5963     }
5964
5965     #start nbd server for storage migration
5966     if (my $nbd = $migrate_opts->{nbd}) {
5967         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5968
5969         my $migrate_storage_uri;
5970         # nbd_protocol_version > 0 for unix socket support
5971         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5972             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5973             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5974             $migrate_storage_uri = "nbd:unix:$socket_path";
5975             $res->{migrate}->{unix_sockets} = [$socket_path];
5976         } else {
5977             my $nodename = nodename();
5978             my $localip = $get_migration_ip->($nodename);
5979             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5980             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5981
5982             mon_cmd($vmid, "nbd-server-start", addr => {
5983                 type => 'inet',
5984                 data => {
5985                     host => "${localip}",
5986                     port => "${storage_migrate_port}",
5987                 },
5988             });
5989             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5990             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5991         }
5992
5993         my $block_info = mon_cmd($vmid, "query-block");
5994         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5995
5996         foreach my $opt (sort keys %$nbd) {
5997             my $drivestr = $nbd->{$opt}->{drivestr};
5998             my $volid = $nbd->{$opt}->{volid};
5999
6000             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
6001
6002             mon_cmd(
6003                 $vmid,
6004                 "block-export-add",
6005                 id => "drive-$opt",
6006                 'node-name' => $block_node,
6007                 writable => JSON::true,
6008                 type => "nbd",
6009                 name => "drive-$opt", # NBD export name
6010             );
6011
6012             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6013             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6014             print "re-using replicated volume: $opt - $volid\n"
6015                 if $nbd->{$opt}->{replicated};
6016
6017             $res->{drives}->{$opt} = $nbd->{$opt};
6018             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6019         }
6020     }
6021
6022     if ($migratedfrom) {
6023         eval {
6024             set_migration_caps($vmid);
6025         };
6026         warn $@ if $@;
6027
6028         if ($spice_port) {
6029             print "spice listens on port $spice_port\n";
6030             $res->{spice_port} = $spice_port;
6031             if ($migrate_opts->{spice_ticket}) {
6032                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6033                     $migrate_opts->{spice_ticket});
6034                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6035             }
6036         }
6037
6038     } else {
6039         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6040             if !$statefile && $conf->{balloon};
6041
6042         foreach my $opt (keys %$conf) {
6043             next if $opt !~  m/^net\d+$/;
6044             my $nicconf = parse_net($conf->{$opt});
6045             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6046         }
6047         add_nets_bridge_fdb($conf, $vmid);
6048     }
6049
6050    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6051         mon_cmd(
6052             $vmid,
6053             'qom-set',
6054             path => "machine/peripheral/balloon0",
6055             property => "guest-stats-polling-interval",
6056             value => 2
6057         );
6058     }
6059
6060     if ($resume) {
6061         print "Resumed VM, removing state\n";
6062         if (my $vmstate = $conf->{vmstate}) {
6063             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6064             PVE::Storage::vdisk_free($storecfg, $vmstate);
6065         }
6066         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6067         PVE::QemuConfig->write_config($vmid, $conf);
6068     }
6069
6070     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6071
6072     return $res;
6073 }
6074
6075 sub vm_commandline {
6076     my ($storecfg, $vmid, $snapname) = @_;
6077
6078     my $conf = PVE::QemuConfig->load_config($vmid);
6079
6080     my ($forcemachine, $forcecpu);
6081     if ($snapname) {
6082         my $snapshot = $conf->{snapshots}->{$snapname};
6083         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6084
6085         # check for machine or CPU overrides in snapshot
6086         $forcemachine = $snapshot->{runningmachine};
6087         $forcecpu = $snapshot->{runningcpu};
6088
6089         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6090
6091         $conf = $snapshot;
6092     }
6093
6094     my $defaults = load_defaults();
6095
6096     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6097
6098     return PVE::Tools::cmd2string($cmd);
6099 }
6100
6101 sub vm_reset {
6102     my ($vmid, $skiplock) = @_;
6103
6104     PVE::QemuConfig->lock_config($vmid, sub {
6105
6106         my $conf = PVE::QemuConfig->load_config($vmid);
6107
6108         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6109
6110         mon_cmd($vmid, "system_reset");
6111     });
6112 }
6113
6114 sub get_vm_volumes {
6115     my ($conf) = @_;
6116
6117     my $vollist = [];
6118     foreach_volid($conf, sub {
6119         my ($volid, $attr) = @_;
6120
6121         return if $volid =~ m|^/|;
6122
6123         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6124         return if !$sid;
6125
6126         push @$vollist, $volid;
6127     });
6128
6129     return $vollist;
6130 }
6131
6132 sub cleanup_pci_devices {
6133     my ($vmid, $conf) = @_;
6134
6135     foreach my $key (keys %$conf) {
6136         next if $key !~ m/^hostpci(\d+)$/;
6137         my $hostpciindex = $1;
6138         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6139         my $d = parse_hostpci($conf->{$key});
6140         if ($d->{mdev}) {
6141             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6142             # don't want to break ABI just for this two liner
6143             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6144             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6145         }
6146     }
6147     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6148 }
6149
6150 sub vm_stop_cleanup {
6151     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6152
6153     eval {
6154
6155         if (!$keepActive) {
6156             my $vollist = get_vm_volumes($conf);
6157             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6158
6159             if (my $tpmdrive = $conf->{tpmstate0}) {
6160                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6161                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6162                 if ($storeid) {
6163                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6164                 }
6165             }
6166         }
6167
6168         foreach my $ext (qw(mon qmp pid vnc qga)) {
6169             unlink "/var/run/qemu-server/${vmid}.$ext";
6170         }
6171
6172         if ($conf->{ivshmem}) {
6173             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6174             # just delete it for now, VMs which have this already open do not
6175             # are affected, but new VMs will get a separated one. If this
6176             # becomes an issue we either add some sort of ref-counting or just
6177             # add a "don't delete on stop" flag to the ivshmem format.
6178             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6179         }
6180
6181         cleanup_pci_devices($vmid, $conf);
6182
6183         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6184     };
6185     warn $@ if $@; # avoid errors - just warn
6186 }
6187
6188 # call only in locked context
6189 sub _do_vm_stop {
6190     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6191
6192     my $pid = check_running($vmid, $nocheck);
6193     return if !$pid;
6194
6195     my $conf;
6196     if (!$nocheck) {
6197         $conf = PVE::QemuConfig->load_config($vmid);
6198         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6199         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6200             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6201             $timeout = $opts->{down} if $opts->{down};
6202         }
6203         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6204     }
6205
6206     eval {
6207         if ($shutdown) {
6208             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6209                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6210             } else {
6211                 mon_cmd($vmid, "system_powerdown");
6212             }
6213         } else {
6214             mon_cmd($vmid, "quit");
6215         }
6216     };
6217     my $err = $@;
6218
6219     if (!$err) {
6220         $timeout = 60 if !defined($timeout);
6221
6222         my $count = 0;
6223         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6224             $count++;
6225             sleep 1;
6226         }
6227
6228         if ($count >= $timeout) {
6229             if ($force) {
6230                 warn "VM still running - terminating now with SIGTERM\n";
6231                 kill 15, $pid;
6232             } else {
6233                 die "VM quit/powerdown failed - got timeout\n";
6234             }
6235         } else {
6236             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6237             return;
6238         }
6239     } else {
6240        if (!check_running($vmid, $nocheck)) {
6241             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6242             return;
6243        }
6244        if ($force) {
6245             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6246             kill 15, $pid;
6247         } else {
6248             die "VM quit/powerdown failed\n";
6249         }
6250     }
6251
6252     # wait again
6253     $timeout = 10;
6254
6255     my $count = 0;
6256     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6257         $count++;
6258         sleep 1;
6259     }
6260
6261     if ($count >= $timeout) {
6262         warn "VM still running - terminating now with SIGKILL\n";
6263         kill 9, $pid;
6264         sleep 1;
6265     }
6266
6267     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6268 }
6269
6270 # Note: use $nocheck to skip tests if VM configuration file exists.
6271 # We need that when migration VMs to other nodes (files already moved)
6272 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6273 sub vm_stop {
6274     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6275
6276     $force = 1 if !defined($force) && !$shutdown;
6277
6278     if ($migratedfrom){
6279         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6280         kill 15, $pid if $pid;
6281         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6282         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6283         return;
6284     }
6285
6286     PVE::QemuConfig->lock_config($vmid, sub {
6287         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6288    });
6289 }
6290
6291 sub vm_reboot {
6292     my ($vmid, $timeout) = @_;
6293
6294     PVE::QemuConfig->lock_config($vmid, sub {
6295         eval {
6296
6297             # only reboot if running, as qmeventd starts it again on a stop event
6298             return if !check_running($vmid);
6299
6300             create_reboot_request($vmid);
6301
6302             my $storecfg = PVE::Storage::config();
6303             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6304
6305         };
6306         if (my $err = $@) {
6307             # avoid that the next normal shutdown will be confused for a reboot
6308             clear_reboot_request($vmid);
6309             die $err;
6310         }
6311    });
6312 }
6313
6314 # note: if using the statestorage parameter, the caller has to check privileges
6315 sub vm_suspend {
6316     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6317
6318     my $conf;
6319     my $path;
6320     my $storecfg;
6321     my $vmstate;
6322
6323     PVE::QemuConfig->lock_config($vmid, sub {
6324
6325         $conf = PVE::QemuConfig->load_config($vmid);
6326
6327         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6328         PVE::QemuConfig->check_lock($conf)
6329             if !($skiplock || $is_backing_up);
6330
6331         die "cannot suspend to disk during backup\n"
6332             if $is_backing_up && $includestate;
6333
6334         if ($includestate) {
6335             $conf->{lock} = 'suspending';
6336             my $date = strftime("%Y-%m-%d", localtime(time()));
6337             $storecfg = PVE::Storage::config();
6338             if (!$statestorage) {
6339                 $statestorage = find_vmstate_storage($conf, $storecfg);
6340                 # check permissions for the storage
6341                 my $rpcenv = PVE::RPCEnvironment::get();
6342                 if ($rpcenv->{type} ne 'cli') {
6343                     my $authuser = $rpcenv->get_user();
6344                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6345                 }
6346             }
6347
6348
6349             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6350                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6351             $path = PVE::Storage::path($storecfg, $vmstate);
6352             PVE::QemuConfig->write_config($vmid, $conf);
6353         } else {
6354             mon_cmd($vmid, "stop");
6355         }
6356     });
6357
6358     if ($includestate) {
6359         # save vm state
6360         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6361
6362         eval {
6363             set_migration_caps($vmid, 1);
6364             mon_cmd($vmid, "savevm-start", statefile => $path);
6365             for(;;) {
6366                 my $state = mon_cmd($vmid, "query-savevm");
6367                 if (!$state->{status}) {
6368                     die "savevm not active\n";
6369                 } elsif ($state->{status} eq 'active') {
6370                     sleep(1);
6371                     next;
6372                 } elsif ($state->{status} eq 'completed') {
6373                     print "State saved, quitting\n";
6374                     last;
6375                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6376                     die "query-savevm failed with error '$state->{error}'\n"
6377                 } else {
6378                     die "query-savevm returned status '$state->{status}'\n";
6379                 }
6380             }
6381         };
6382         my $err = $@;
6383
6384         PVE::QemuConfig->lock_config($vmid, sub {
6385             $conf = PVE::QemuConfig->load_config($vmid);
6386             if ($err) {
6387                 # cleanup, but leave suspending lock, to indicate something went wrong
6388                 eval {
6389                     mon_cmd($vmid, "savevm-end");
6390                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6391                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6392                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6393                     PVE::QemuConfig->write_config($vmid, $conf);
6394                 };
6395                 warn $@ if $@;
6396                 die $err;
6397             }
6398
6399             die "lock changed unexpectedly\n"
6400                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6401
6402             mon_cmd($vmid, "quit");
6403             $conf->{lock} = 'suspended';
6404             PVE::QemuConfig->write_config($vmid, $conf);
6405         });
6406     }
6407 }
6408
6409 # $nocheck is set when called as part of a migration - in this context the
6410 # location of the config file (source or target node) is not deterministic,
6411 # since migration cannot wait for pmxcfs to process the rename
6412 sub vm_resume {
6413     my ($vmid, $skiplock, $nocheck) = @_;
6414
6415     PVE::QemuConfig->lock_config($vmid, sub {
6416         my $res = mon_cmd($vmid, 'query-status');
6417         my $resume_cmd = 'cont';
6418         my $reset = 0;
6419         my $conf;
6420         if ($nocheck) {
6421             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6422             if ($@) {
6423                 my $vmlist = PVE::Cluster::get_vmlist();
6424                 if (exists($vmlist->{ids}->{$vmid})) {
6425                     my $node = $vmlist->{ids}->{$vmid}->{node};
6426                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6427                 }
6428                 if (!$conf) {
6429                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6430                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6431                 }
6432             }
6433         } else {
6434             $conf = PVE::QemuConfig->load_config($vmid);
6435         }
6436
6437         if ($res->{status}) {
6438             return if $res->{status} eq 'running'; # job done, go home
6439             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6440             $reset = 1 if $res->{status} eq 'shutdown';
6441         }
6442
6443         if (!$nocheck) {
6444             PVE::QemuConfig->check_lock($conf)
6445                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6446         }
6447
6448         if ($reset) {
6449             # required if a VM shuts down during a backup and we get a resume
6450             # request before the backup finishes for example
6451             mon_cmd($vmid, "system_reset");
6452         }
6453
6454         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6455
6456         mon_cmd($vmid, $resume_cmd);
6457     });
6458 }
6459
6460 sub vm_sendkey {
6461     my ($vmid, $skiplock, $key) = @_;
6462
6463     PVE::QemuConfig->lock_config($vmid, sub {
6464
6465         my $conf = PVE::QemuConfig->load_config($vmid);
6466
6467         # there is no qmp command, so we use the human monitor command
6468         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6469         die $res if $res ne '';
6470     });
6471 }
6472
6473 # vzdump restore implementaion
6474
6475 sub tar_archive_read_firstfile {
6476     my $archive = shift;
6477
6478     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6479
6480     # try to detect archive type first
6481     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6482         die "unable to open file '$archive'\n";
6483     my $firstfile = <$fh>;
6484     kill 15, $pid;
6485     close $fh;
6486
6487     die "ERROR: archive contaions no data\n" if !$firstfile;
6488     chomp $firstfile;
6489
6490     return $firstfile;
6491 }
6492
6493 sub tar_restore_cleanup {
6494     my ($storecfg, $statfile) = @_;
6495
6496     print STDERR "starting cleanup\n";
6497
6498     if (my $fd = IO::File->new($statfile, "r")) {
6499         while (defined(my $line = <$fd>)) {
6500             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6501                 my $volid = $2;
6502                 eval {
6503                     if ($volid =~ m|^/|) {
6504                         unlink $volid || die 'unlink failed\n';
6505                     } else {
6506                         PVE::Storage::vdisk_free($storecfg, $volid);
6507                     }
6508                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6509                 };
6510                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6511             } else {
6512                 print STDERR "unable to parse line in statfile - $line";
6513             }
6514         }
6515         $fd->close();
6516     }
6517 }
6518
6519 sub restore_file_archive {
6520     my ($archive, $vmid, $user, $opts) = @_;
6521
6522     return restore_vma_archive($archive, $vmid, $user, $opts)
6523         if $archive eq '-';
6524
6525     my $info = PVE::Storage::archive_info($archive);
6526     my $format = $opts->{format} // $info->{format};
6527     my $comp = $info->{compression};
6528
6529     # try to detect archive format
6530     if ($format eq 'tar') {
6531         return restore_tar_archive($archive, $vmid, $user, $opts);
6532     } else {
6533         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6534     }
6535 }
6536
6537 # hepler to remove disks that will not be used after restore
6538 my $restore_cleanup_oldconf = sub {
6539     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6540
6541     my $kept_disks = {};
6542
6543     PVE::QemuConfig->foreach_volume($oldconf, sub {
6544         my ($ds, $drive) = @_;
6545
6546         return if drive_is_cdrom($drive, 1);
6547
6548         my $volid = $drive->{file};
6549         return if !$volid || $volid =~ m|^/|;
6550
6551         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6552         return if !$path || !$owner || ($owner != $vmid);
6553
6554         # Note: only delete disk we want to restore
6555         # other volumes will become unused
6556         if ($virtdev_hash->{$ds}) {
6557             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6558             if (my $err = $@) {
6559                 warn $err;
6560             }
6561         } else {
6562             $kept_disks->{$volid} = 1;
6563         }
6564     });
6565
6566     # after the restore we have no snapshots anymore
6567     for my $snapname (keys $oldconf->{snapshots}->%*) {
6568         my $snap = $oldconf->{snapshots}->{$snapname};
6569         if ($snap->{vmstate}) {
6570             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6571             if (my $err = $@) {
6572                 warn $err;
6573             }
6574         }
6575
6576         for my $volid (keys $kept_disks->%*) {
6577             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6578             warn $@ if $@;
6579         }
6580     }
6581 };
6582
6583 # Helper to parse vzdump backup device hints
6584 #
6585 # $rpcenv: Environment, used to ckeck storage permissions
6586 # $user: User ID, to check storage permissions
6587 # $storecfg: Storage configuration
6588 # $fh: the file handle for reading the configuration
6589 # $devinfo: should contain device sizes for all backu-up'ed devices
6590 # $options: backup options (pool, default storage)
6591 #
6592 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6593 my $parse_backup_hints = sub {
6594     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6595
6596     my $check_storage = sub { # assert if an image can be allocate
6597         my ($storeid, $scfg) = @_;
6598         die "Content type 'images' is not available on storage '$storeid'\n"
6599             if !$scfg->{content}->{images};
6600         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6601             if $user ne 'root@pam';
6602     };
6603
6604     my $virtdev_hash = {};
6605     while (defined(my $line = <$fh>)) {
6606         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6607             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6608             die "archive does not contain data for drive '$virtdev'\n"
6609                 if !$devinfo->{$devname};
6610
6611             if (defined($options->{storage})) {
6612                 $storeid = $options->{storage} || 'local';
6613             } elsif (!$storeid) {
6614                 $storeid = 'local';
6615             }
6616             $format = 'raw' if !$format;
6617             $devinfo->{$devname}->{devname} = $devname;
6618             $devinfo->{$devname}->{virtdev} = $virtdev;
6619             $devinfo->{$devname}->{format} = $format;
6620             $devinfo->{$devname}->{storeid} = $storeid;
6621
6622             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6623             $check_storage->($storeid, $scfg); # permission and content type check
6624
6625             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6626         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6627             my $virtdev = $1;
6628             my $drive = parse_drive($virtdev, $2);
6629
6630             if (drive_is_cloudinit($drive)) {
6631                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6632                 $storeid = $options->{storage} if defined ($options->{storage});
6633                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6634                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6635
6636                 $check_storage->($storeid, $scfg); # permission and content type check
6637
6638                 $virtdev_hash->{$virtdev} = {
6639                     format => $format,
6640                     storeid => $storeid,
6641                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6642                     is_cloudinit => 1,
6643                 };
6644             }
6645         }
6646     }
6647
6648     return $virtdev_hash;
6649 };
6650
6651 # Helper to allocate and activate all volumes required for a restore
6652 #
6653 # $storecfg: Storage configuration
6654 # $virtdev_hash: as returned by parse_backup_hints()
6655 #
6656 # Returns: { $virtdev => $volid }
6657 my $restore_allocate_devices = sub {
6658     my ($storecfg, $virtdev_hash, $vmid) = @_;
6659
6660     my $map = {};
6661     foreach my $virtdev (sort keys %$virtdev_hash) {
6662         my $d = $virtdev_hash->{$virtdev};
6663         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6664         my $storeid = $d->{storeid};
6665         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6666
6667         # test if requested format is supported
6668         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6669         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6670         $d->{format} = $defFormat if !$supported;
6671
6672         my $name;
6673         if ($d->{is_cloudinit}) {
6674             $name = "vm-$vmid-cloudinit";
6675             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6676             if ($scfg->{path}) {
6677                 $name .= ".$d->{format}";
6678             }
6679         }
6680
6681         my $volid = PVE::Storage::vdisk_alloc(
6682             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6683
6684         print STDERR "new volume ID is '$volid'\n";
6685         $d->{volid} = $volid;
6686
6687         PVE::Storage::activate_volumes($storecfg, [$volid]);
6688
6689         $map->{$virtdev} = $volid;
6690     }
6691
6692     return $map;
6693 };
6694
6695 sub restore_update_config_line {
6696     my ($cookie, $map, $line, $unique) = @_;
6697
6698     return '' if $line =~ m/^\#qmdump\#/;
6699     return '' if $line =~ m/^\#vzdump\#/;
6700     return '' if $line =~ m/^lock:/;
6701     return '' if $line =~ m/^unused\d+:/;
6702     return '' if $line =~ m/^parent:/;
6703
6704     my $res = '';
6705
6706     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6707     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6708         # try to convert old 1.X settings
6709         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6710         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6711             my ($model, $macaddr) = split(/\=/, $devconfig);
6712             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6713             my $net = {
6714                 model => $model,
6715                 bridge => "vmbr$ind",
6716                 macaddr => $macaddr,
6717             };
6718             my $netstr = print_net($net);
6719
6720             $res .= "net$cookie->{netcount}: $netstr\n";
6721             $cookie->{netcount}++;
6722         }
6723     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6724         my ($id, $netstr) = ($1, $2);
6725         my $net = parse_net($netstr);
6726         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6727         $netstr = print_net($net);
6728         $res .= "$id: $netstr\n";
6729     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6730         my $virtdev = $1;
6731         my $value = $3;
6732         my $di = parse_drive($virtdev, $value);
6733         if (defined($di->{backup}) && !$di->{backup}) {
6734             $res .= "#$line";
6735         } elsif ($map->{$virtdev}) {
6736             delete $di->{format}; # format can change on restore
6737             $di->{file} = $map->{$virtdev};
6738             $value = print_drive($di);
6739             $res .= "$virtdev: $value\n";
6740         } else {
6741             $res .= $line;
6742         }
6743     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6744         my $vmgenid = $1;
6745         if ($vmgenid ne '0') {
6746             # always generate a new vmgenid if there was a valid one setup
6747             $vmgenid = generate_uuid();
6748         }
6749         $res .= "vmgenid: $vmgenid\n";
6750     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6751         my ($uuid, $uuid_str);
6752         UUID::generate($uuid);
6753         UUID::unparse($uuid, $uuid_str);
6754         my $smbios1 = parse_smbios1($2);
6755         $smbios1->{uuid} = $uuid_str;
6756         $res .= $1.print_smbios1($smbios1)."\n";
6757     } else {
6758         $res .= $line;
6759     }
6760
6761     return $res;
6762 }
6763
6764 my $restore_deactivate_volumes = sub {
6765     my ($storecfg, $virtdev_hash) = @_;
6766
6767     my $vollist = [];
6768     for my $dev (values $virtdev_hash->%*) {
6769         push $vollist->@*, $dev->{volid} if $dev->{volid};
6770     }
6771
6772     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6773     print STDERR $@ if $@;
6774 };
6775
6776 my $restore_destroy_volumes = sub {
6777     my ($storecfg, $virtdev_hash) = @_;
6778
6779     for my $dev (values $virtdev_hash->%*) {
6780         my $volid = $dev->{volid} or next;
6781         eval {
6782             PVE::Storage::vdisk_free($storecfg, $volid);
6783             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6784         };
6785         print STDERR "unable to cleanup '$volid' - $@" if $@;
6786     }
6787 };
6788
6789 my $restore_merge_config = sub {
6790     my ($filename, $backup_conf_raw, $override_conf) = @_;
6791
6792     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6793     for my $key (keys $override_conf->%*) {
6794         $backup_conf->{$key} = $override_conf->{$key};
6795     }
6796
6797     return $backup_conf;
6798 };
6799
6800 sub scan_volids {
6801     my ($cfg, $vmid) = @_;
6802
6803     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6804
6805     my $volid_hash = {};
6806     foreach my $storeid (keys %$info) {
6807         foreach my $item (@{$info->{$storeid}}) {
6808             next if !($item->{volid} && $item->{size});
6809             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6810             $volid_hash->{$item->{volid}} = $item;
6811         }
6812     }
6813
6814     return $volid_hash;
6815 }
6816
6817 sub update_disk_config {
6818     my ($vmid, $conf, $volid_hash) = @_;
6819
6820     my $changes;
6821     my $prefix = "VM $vmid";
6822
6823     # used and unused disks
6824     my $referenced = {};
6825
6826     # Note: it is allowed to define multiple storages with same path (alias), so
6827     # we need to check both 'volid' and real 'path' (two different volid can point
6828     # to the same path).
6829
6830     my $referencedpath = {};
6831
6832     # update size info
6833     PVE::QemuConfig->foreach_volume($conf, sub {
6834         my ($opt, $drive) = @_;
6835
6836         my $volid = $drive->{file};
6837         return if !$volid;
6838         my $volume = $volid_hash->{$volid};
6839
6840         # mark volid as "in-use" for next step
6841         $referenced->{$volid} = 1;
6842         if ($volume && (my $path = $volume->{path})) {
6843             $referencedpath->{$path} = 1;
6844         }
6845
6846         return if drive_is_cdrom($drive);
6847         return if !$volume;
6848
6849         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6850         if (defined($updated)) {
6851             $changes = 1;
6852             $conf->{$opt} = print_drive($updated);
6853             print "$prefix ($opt): $msg\n";
6854         }
6855     });
6856
6857     # remove 'unusedX' entry if volume is used
6858     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6859         my ($opt, $drive) = @_;
6860
6861         my $volid = $drive->{file};
6862         return if !$volid;
6863
6864         my $path;
6865         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6866         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6867             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6868             $changes = 1;
6869             delete $conf->{$opt};
6870         }
6871
6872         $referenced->{$volid} = 1;
6873         $referencedpath->{$path} = 1 if $path;
6874     });
6875
6876     foreach my $volid (sort keys %$volid_hash) {
6877         next if $volid =~ m/vm-$vmid-state-/;
6878         next if $referenced->{$volid};
6879         my $path = $volid_hash->{$volid}->{path};
6880         next if !$path; # just to be sure
6881         next if $referencedpath->{$path};
6882         $changes = 1;
6883         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6884         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6885         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6886     }
6887
6888     return $changes;
6889 }
6890
6891 sub rescan {
6892     my ($vmid, $nolock, $dryrun) = @_;
6893
6894     my $cfg = PVE::Storage::config();
6895
6896     print "rescan volumes...\n";
6897     my $volid_hash = scan_volids($cfg, $vmid);
6898
6899     my $updatefn =  sub {
6900         my ($vmid) = @_;
6901
6902         my $conf = PVE::QemuConfig->load_config($vmid);
6903
6904         PVE::QemuConfig->check_lock($conf);
6905
6906         my $vm_volids = {};
6907         foreach my $volid (keys %$volid_hash) {
6908             my $info = $volid_hash->{$volid};
6909             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6910         }
6911
6912         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6913
6914         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6915     };
6916
6917     if (defined($vmid)) {
6918         if ($nolock) {
6919             &$updatefn($vmid);
6920         } else {
6921             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6922         }
6923     } else {
6924         my $vmlist = config_list();
6925         foreach my $vmid (keys %$vmlist) {
6926             if ($nolock) {
6927                 &$updatefn($vmid);
6928             } else {
6929                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6930             }
6931         }
6932     }
6933 }
6934
6935 sub restore_proxmox_backup_archive {
6936     my ($archive, $vmid, $user, $options) = @_;
6937
6938     my $storecfg = PVE::Storage::config();
6939
6940     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6941     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6942
6943     my $fingerprint = $scfg->{fingerprint};
6944     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6945
6946     my $repo = PVE::PBSClient::get_repository($scfg);
6947     my $namespace = $scfg->{namespace};
6948
6949     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6950     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6951     local $ENV{PBS_PASSWORD} = $password;
6952     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6953
6954     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6955         PVE::Storage::parse_volname($storecfg, $archive);
6956
6957     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6958
6959     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6960
6961     my $tmpdir = "/var/tmp/vzdumptmp$$";
6962     rmtree $tmpdir;
6963     mkpath $tmpdir;
6964
6965     my $conffile = PVE::QemuConfig->config_file($vmid);
6966      # disable interrupts (always do cleanups)
6967     local $SIG{INT} =
6968         local $SIG{TERM} =
6969         local $SIG{QUIT} =
6970         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6971
6972     # Note: $oldconf is undef if VM does not exists
6973     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6974     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6975     my $new_conf_raw = '';
6976
6977     my $rpcenv = PVE::RPCEnvironment::get();
6978     my $devinfo = {}; # info about drives included in backup
6979     my $virtdev_hash = {}; # info about allocated drives
6980
6981     eval {
6982         # enable interrupts
6983         local $SIG{INT} =
6984             local $SIG{TERM} =
6985             local $SIG{QUIT} =
6986             local $SIG{HUP} =
6987             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6988
6989         my $cfgfn = "$tmpdir/qemu-server.conf";
6990         my $firewall_config_fn = "$tmpdir/fw.conf";
6991         my $index_fn = "$tmpdir/index.json";
6992
6993         my $cmd = "restore";
6994
6995         my $param = [$pbs_backup_name, "index.json", $index_fn];
6996         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6997         my $index = PVE::Tools::file_get_contents($index_fn);
6998         $index = decode_json($index);
6999
7000         foreach my $info (@{$index->{files}}) {
7001             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7002                 my $devname = $1;
7003                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7004                     $devinfo->{$devname}->{size} = $1;
7005                 } else {
7006                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7007                 }
7008             }
7009         }
7010
7011         my $is_qemu_server_backup = scalar(
7012             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7013         );
7014         if (!$is_qemu_server_backup) {
7015             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7016         }
7017         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7018
7019         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7020         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7021
7022         if ($has_firewall_config) {
7023             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7024             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7025
7026             my $pve_firewall_dir = '/etc/pve/firewall';
7027             mkdir $pve_firewall_dir; # make sure the dir exists
7028             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7029         }
7030
7031         my $fh = IO::File->new($cfgfn, "r") ||
7032             die "unable to read qemu-server.conf - $!\n";
7033
7034         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7035
7036         # fixme: rate limit?
7037
7038         # create empty/temp config
7039         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7040
7041         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7042
7043         # allocate volumes
7044         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7045
7046         foreach my $virtdev (sort keys %$virtdev_hash) {
7047             my $d = $virtdev_hash->{$virtdev};
7048             next if $d->{is_cloudinit}; # no need to restore cloudinit
7049
7050             # this fails if storage is unavailable
7051             my $volid = $d->{volid};
7052             my $path = PVE::Storage::path($storecfg, $volid);
7053
7054             # for live-restore we only want to preload the efidisk and TPM state
7055             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7056
7057             my @ns_arg;
7058             if (defined(my $ns = $scfg->{namespace})) {
7059                 @ns_arg = ('--ns', $ns);
7060             }
7061
7062             my $pbs_restore_cmd = [
7063                 '/usr/bin/pbs-restore',
7064                 '--repository', $repo,
7065                 @ns_arg,
7066                 $pbs_backup_name,
7067                 "$d->{devname}.img.fidx",
7068                 $path,
7069                 '--verbose',
7070                 ];
7071
7072             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7073             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7074
7075             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7076                 push @$pbs_restore_cmd, '--skip-zero';
7077             }
7078
7079             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7080             print "restore proxmox backup image: $dbg_cmdstring\n";
7081             run_command($pbs_restore_cmd);
7082         }
7083
7084         $fh->seek(0, 0) || die "seek failed - $!\n";
7085
7086         my $cookie = { netcount => 0 };
7087         while (defined(my $line = <$fh>)) {
7088             $new_conf_raw .= restore_update_config_line(
7089                 $cookie,
7090                 $map,
7091                 $line,
7092                 $options->{unique},
7093             );
7094         }
7095
7096         $fh->close();
7097     };
7098     my $err = $@;
7099
7100     if ($err || !$options->{live}) {
7101         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7102     }
7103
7104     rmtree $tmpdir;
7105
7106     if ($err) {
7107         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7108         die $err;
7109     }
7110
7111     if ($options->{live}) {
7112         # keep lock during live-restore
7113         $new_conf_raw .= "\nlock: create";
7114     }
7115
7116     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7117     PVE::QemuConfig->write_config($vmid, $new_conf);
7118
7119     eval { rescan($vmid, 1); };
7120     warn $@ if $@;
7121
7122     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7123
7124     if ($options->{live}) {
7125         # enable interrupts
7126         local $SIG{INT} =
7127             local $SIG{TERM} =
7128             local $SIG{QUIT} =
7129             local $SIG{HUP} =
7130             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7131
7132         my $conf = PVE::QemuConfig->load_config($vmid);
7133         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7134
7135         # these special drives are already restored before start
7136         delete $devinfo->{'drive-efidisk0'};
7137         delete $devinfo->{'drive-tpmstate0-backup'};
7138
7139         my $pbs_opts = {
7140             repo => $repo,
7141             keyfile => $keyfile,
7142             snapshot => $pbs_backup_name,
7143             namespace => $namespace,
7144         };
7145         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7146
7147         PVE::QemuConfig->remove_lock($vmid, "create");
7148     }
7149 }
7150
7151 sub pbs_live_restore {
7152     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7153
7154     print "starting VM for live-restore\n";
7155     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7156
7157     my $pbs_backing = {};
7158     for my $ds (keys %$restored_disks) {
7159         $ds =~ m/^drive-(.*)$/;
7160         my $confname = $1;
7161         $pbs_backing->{$confname} = {
7162             repository => $opts->{repo},
7163             snapshot => $opts->{snapshot},
7164             archive => "$ds.img.fidx",
7165         };
7166         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7167         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7168
7169         my $drive = parse_drive($confname, $conf->{$confname});
7170         print "restoring '$ds' to '$drive->{file}'\n";
7171     }
7172
7173     my $drives_streamed = 0;
7174     eval {
7175         # make sure HA doesn't interrupt our restore by stopping the VM
7176         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7177             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7178         }
7179
7180         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7181         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7182         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7183
7184         my $qmeventd_fd = register_qmeventd_handle($vmid);
7185
7186         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7187         # this will effectively collapse the backing image chain consisting of
7188         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7189         # removes itself once all backing images vanish with 'auto-remove=on')
7190         my $jobs = {};
7191         for my $ds (sort keys %$restored_disks) {
7192             my $job_id = "restore-$ds";
7193             mon_cmd($vmid, 'block-stream',
7194                 'job-id' => $job_id,
7195                 device => "$ds",
7196             );
7197             $jobs->{$job_id} = {};
7198         }
7199
7200         mon_cmd($vmid, 'cont');
7201         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7202
7203         print "restore-drive jobs finished successfully, removing all tracking block devices"
7204             ." to disconnect from Proxmox Backup Server\n";
7205
7206         for my $ds (sort keys %$restored_disks) {
7207             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7208         }
7209
7210         close($qmeventd_fd);
7211     };
7212
7213     my $err = $@;
7214
7215     if ($err) {
7216         warn "An error occurred during live-restore: $err\n";
7217         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7218         die "live-restore failed\n";
7219     }
7220 }
7221
7222 sub restore_vma_archive {
7223     my ($archive, $vmid, $user, $opts, $comp) = @_;
7224
7225     my $readfrom = $archive;
7226
7227     my $cfg = PVE::Storage::config();
7228     my $commands = [];
7229     my $bwlimit = $opts->{bwlimit};
7230
7231     my $dbg_cmdstring = '';
7232     my $add_pipe = sub {
7233         my ($cmd) = @_;
7234         push @$commands, $cmd;
7235         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7236         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7237         $readfrom = '-';
7238     };
7239
7240     my $input = undef;
7241     if ($archive eq '-') {
7242         $input = '<&STDIN';
7243     } else {
7244         # If we use a backup from a PVE defined storage we also consider that
7245         # storage's rate limit:
7246         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7247         if (defined($volid)) {
7248             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7249             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7250             if ($readlimit) {
7251                 print STDERR "applying read rate limit: $readlimit\n";
7252                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7253                 $add_pipe->($cstream);
7254             }
7255         }
7256     }
7257
7258     if ($comp) {
7259         my $info = PVE::Storage::decompressor_info('vma', $comp);
7260         my $cmd = $info->{decompressor};
7261         push @$cmd, $readfrom;
7262         $add_pipe->($cmd);
7263     }
7264
7265     my $tmpdir = "/var/tmp/vzdumptmp$$";
7266     rmtree $tmpdir;
7267
7268     # disable interrupts (always do cleanups)
7269     local $SIG{INT} =
7270         local $SIG{TERM} =
7271         local $SIG{QUIT} =
7272         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7273
7274     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7275     POSIX::mkfifo($mapfifo, 0600);
7276     my $fifofh;
7277     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7278
7279     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7280
7281     my $oldtimeout;
7282     my $timeout = 5;
7283
7284     my $devinfo = {}; # info about drives included in backup
7285     my $virtdev_hash = {}; # info about allocated drives
7286
7287     my $rpcenv = PVE::RPCEnvironment::get();
7288
7289     my $conffile = PVE::QemuConfig->config_file($vmid);
7290
7291     # Note: $oldconf is undef if VM does not exist
7292     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7293     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7294     my $new_conf_raw = '';
7295
7296     my %storage_limits;
7297
7298     my $print_devmap = sub {
7299         my $cfgfn = "$tmpdir/qemu-server.conf";
7300
7301         # we can read the config - that is already extracted
7302         my $fh = IO::File->new($cfgfn, "r") ||
7303             die "unable to read qemu-server.conf - $!\n";
7304
7305         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7306         if (-f $fwcfgfn) {
7307             my $pve_firewall_dir = '/etc/pve/firewall';
7308             mkdir $pve_firewall_dir; # make sure the dir exists
7309             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7310         }
7311
7312         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7313
7314         foreach my $info (values %{$virtdev_hash}) {
7315             my $storeid = $info->{storeid};
7316             next if defined($storage_limits{$storeid});
7317
7318             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7319             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7320             $storage_limits{$storeid} = $limit * 1024;
7321         }
7322
7323         foreach my $devname (keys %$devinfo) {
7324             die "found no device mapping information for device '$devname'\n"
7325                 if !$devinfo->{$devname}->{virtdev};
7326         }
7327
7328         # create empty/temp config
7329         if ($oldconf) {
7330             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7331             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7332         }
7333
7334         # allocate volumes
7335         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7336
7337         # print restore information to $fifofh
7338         foreach my $virtdev (sort keys %$virtdev_hash) {
7339             my $d = $virtdev_hash->{$virtdev};
7340             next if $d->{is_cloudinit}; # no need to restore cloudinit
7341
7342             my $storeid = $d->{storeid};
7343             my $volid = $d->{volid};
7344
7345             my $map_opts = '';
7346             if (my $limit = $storage_limits{$storeid}) {
7347                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7348             }
7349
7350             my $write_zeros = 1;
7351             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7352                 $write_zeros = 0;
7353             }
7354
7355             my $path = PVE::Storage::path($cfg, $volid);
7356
7357             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7358
7359             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7360         }
7361
7362         $fh->seek(0, 0) || die "seek failed - $!\n";
7363
7364         my $cookie = { netcount => 0 };
7365         while (defined(my $line = <$fh>)) {
7366             $new_conf_raw .= restore_update_config_line(
7367                 $cookie,
7368                 $map,
7369                 $line,
7370                 $opts->{unique},
7371             );
7372         }
7373
7374         $fh->close();
7375     };
7376
7377     eval {
7378         # enable interrupts
7379         local $SIG{INT} =
7380             local $SIG{TERM} =
7381             local $SIG{QUIT} =
7382             local $SIG{HUP} =
7383             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7384         local $SIG{ALRM} = sub { die "got timeout\n"; };
7385
7386         $oldtimeout = alarm($timeout);
7387
7388         my $parser = sub {
7389             my $line = shift;
7390
7391             print "$line\n";
7392
7393             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7394                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7395                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7396             } elsif ($line =~ m/^CTIME: /) {
7397                 # we correctly received the vma config, so we can disable
7398                 # the timeout now for disk allocation (set to 10 minutes, so
7399                 # that we always timeout if something goes wrong)
7400                 alarm(600);
7401                 &$print_devmap();
7402                 print $fifofh "done\n";
7403                 my $tmp = $oldtimeout || 0;
7404                 $oldtimeout = undef;
7405                 alarm($tmp);
7406                 close($fifofh);
7407                 $fifofh = undef;
7408             }
7409         };
7410
7411         print "restore vma archive: $dbg_cmdstring\n";
7412         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7413     };
7414     my $err = $@;
7415
7416     alarm($oldtimeout) if $oldtimeout;
7417
7418     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7419
7420     close($fifofh) if $fifofh;
7421     unlink $mapfifo;
7422     rmtree $tmpdir;
7423
7424     if ($err) {
7425         $restore_destroy_volumes->($cfg, $virtdev_hash);
7426         die $err;
7427     }
7428
7429     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7430     PVE::QemuConfig->write_config($vmid, $new_conf);
7431
7432     eval { rescan($vmid, 1); };
7433     warn $@ if $@;
7434
7435     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7436 }
7437
7438 sub restore_tar_archive {
7439     my ($archive, $vmid, $user, $opts) = @_;
7440
7441     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7442         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7443         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7444     }
7445
7446     if ($archive ne '-') {
7447         my $firstfile = tar_archive_read_firstfile($archive);
7448         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7449             if $firstfile ne 'qemu-server.conf';
7450     }
7451
7452     my $storecfg = PVE::Storage::config();
7453
7454     # avoid zombie disks when restoring over an existing VM -> cleanup first
7455     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7456     # skiplock=1 because qmrestore has set the 'create' lock itself already
7457     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7458     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7459
7460     my $tocmd = "/usr/lib/qemu-server/qmextract";
7461
7462     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7463     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7464     $tocmd .= ' --prealloc' if $opts->{prealloc};
7465     $tocmd .= ' --info' if $opts->{info};
7466
7467     # tar option "xf" does not autodetect compression when read from STDIN,
7468     # so we pipe to zcat
7469     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7470         PVE::Tools::shellquote("--to-command=$tocmd");
7471
7472     my $tmpdir = "/var/tmp/vzdumptmp$$";
7473     mkpath $tmpdir;
7474
7475     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7476     local $ENV{VZDUMP_VMID} = $vmid;
7477     local $ENV{VZDUMP_USER} = $user;
7478
7479     my $conffile = PVE::QemuConfig->config_file($vmid);
7480     my $new_conf_raw = '';
7481
7482     # disable interrupts (always do cleanups)
7483     local $SIG{INT} =
7484         local $SIG{TERM} =
7485         local $SIG{QUIT} =
7486         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7487
7488     eval {
7489         # enable interrupts
7490         local $SIG{INT} =
7491             local $SIG{TERM} =
7492             local $SIG{QUIT} =
7493             local $SIG{HUP} =
7494             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7495
7496         if ($archive eq '-') {
7497             print "extracting archive from STDIN\n";
7498             run_command($cmd, input => "<&STDIN");
7499         } else {
7500             print "extracting archive '$archive'\n";
7501             run_command($cmd);
7502         }
7503
7504         return if $opts->{info};
7505
7506         # read new mapping
7507         my $map = {};
7508         my $statfile = "$tmpdir/qmrestore.stat";
7509         if (my $fd = IO::File->new($statfile, "r")) {
7510             while (defined (my $line = <$fd>)) {
7511                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7512                     $map->{$1} = $2 if $1;
7513                 } else {
7514                     print STDERR "unable to parse line in statfile - $line\n";
7515                 }
7516             }
7517             $fd->close();
7518         }
7519
7520         my $confsrc = "$tmpdir/qemu-server.conf";
7521
7522         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7523
7524         my $cookie = { netcount => 0 };
7525         while (defined (my $line = <$srcfd>)) {
7526             $new_conf_raw .= restore_update_config_line(
7527                 $cookie,
7528                 $map,
7529                 $line,
7530                 $opts->{unique},
7531             );
7532         }
7533
7534         $srcfd->close();
7535     };
7536     if (my $err = $@) {
7537         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7538         die $err;
7539     }
7540
7541     rmtree $tmpdir;
7542
7543     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7544
7545     PVE::Cluster::cfs_update(); # make sure we read new file
7546
7547     eval { rescan($vmid, 1); };
7548     warn $@ if $@;
7549 };
7550
7551 sub foreach_storage_used_by_vm {
7552     my ($conf, $func) = @_;
7553
7554     my $sidhash = {};
7555
7556     PVE::QemuConfig->foreach_volume($conf, sub {
7557         my ($ds, $drive) = @_;
7558         return if drive_is_cdrom($drive);
7559
7560         my $volid = $drive->{file};
7561
7562         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7563         $sidhash->{$sid} = $sid if $sid;
7564     });
7565
7566     foreach my $sid (sort keys %$sidhash) {
7567         &$func($sid);
7568     }
7569 }
7570
7571 my $qemu_snap_storage = {
7572     rbd => 1,
7573 };
7574 sub do_snapshots_with_qemu {
7575     my ($storecfg, $volid, $deviceid) = @_;
7576
7577     return if $deviceid =~ m/tpmstate0/;
7578
7579     my $storage_name = PVE::Storage::parse_volume_id($volid);
7580     my $scfg = $storecfg->{ids}->{$storage_name};
7581     die "could not find storage '$storage_name'\n" if !defined($scfg);
7582
7583     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7584         return 1;
7585     }
7586
7587     if ($volid =~ m/\.(qcow2|qed)$/){
7588         return 1;
7589     }
7590
7591     return;
7592 }
7593
7594 sub qga_check_running {
7595     my ($vmid, $nowarn) = @_;
7596
7597     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7598     if ($@) {
7599         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7600         return 0;
7601     }
7602     return 1;
7603 }
7604
7605 sub template_create {
7606     my ($vmid, $conf, $disk) = @_;
7607
7608     my $storecfg = PVE::Storage::config();
7609
7610     PVE::QemuConfig->foreach_volume($conf, sub {
7611         my ($ds, $drive) = @_;
7612
7613         return if drive_is_cdrom($drive);
7614         return if $disk && $ds ne $disk;
7615
7616         my $volid = $drive->{file};
7617         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7618
7619         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7620         $drive->{file} = $voliddst;
7621         $conf->{$ds} = print_drive($drive);
7622         PVE::QemuConfig->write_config($vmid, $conf);
7623     });
7624 }
7625
7626 sub convert_iscsi_path {
7627     my ($path) = @_;
7628
7629     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7630         my $portal = $1;
7631         my $target = $2;
7632         my $lun = $3;
7633
7634         my $initiator_name = get_initiator_name();
7635
7636         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7637                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7638     }
7639
7640     die "cannot convert iscsi path '$path', unkown format\n";
7641 }
7642
7643 sub qemu_img_convert {
7644     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7645
7646     my $storecfg = PVE::Storage::config();
7647     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7648     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7649
7650     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7651
7652     my $cachemode;
7653     my $src_path;
7654     my $src_is_iscsi = 0;
7655     my $src_format;
7656
7657     if ($src_storeid) {
7658         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7659         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7660         $src_format = qemu_img_format($src_scfg, $src_volname);
7661         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7662         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7663         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7664     } elsif (-f $src_volid || -b $src_volid) {
7665         $src_path = $src_volid;
7666         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7667             $src_format = $1;
7668         }
7669     }
7670
7671     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7672
7673     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7674     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7675     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7676     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7677
7678     my $cmd = [];
7679     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7680     push @$cmd, '-l', "snapshot.name=$snapname"
7681         if $snapname && $src_format && $src_format eq "qcow2";
7682     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7683     push @$cmd, '-T', $cachemode if defined($cachemode);
7684
7685     if ($src_is_iscsi) {
7686         push @$cmd, '--image-opts';
7687         $src_path = convert_iscsi_path($src_path);
7688     } elsif ($src_format) {
7689         push @$cmd, '-f', $src_format;
7690     }
7691
7692     if ($dst_is_iscsi) {
7693         push @$cmd, '--target-image-opts';
7694         $dst_path = convert_iscsi_path($dst_path);
7695     } else {
7696         push @$cmd, '-O', $dst_format;
7697     }
7698
7699     push @$cmd, $src_path;
7700
7701     if (!$dst_is_iscsi && $is_zero_initialized) {
7702         push @$cmd, "zeroinit:$dst_path";
7703     } else {
7704         push @$cmd, $dst_path;
7705     }
7706
7707     my $parser = sub {
7708         my $line = shift;
7709         if($line =~ m/\((\S+)\/100\%\)/){
7710             my $percent = $1;
7711             my $transferred = int($size * $percent / 100);
7712             my $total_h = render_bytes($size, 1);
7713             my $transferred_h = render_bytes($transferred, 1);
7714
7715             print "transferred $transferred_h of $total_h ($percent%)\n";
7716         }
7717
7718     };
7719
7720     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7721     my $err = $@;
7722     die "copy failed: $err" if $err;
7723 }
7724
7725 sub qemu_img_format {
7726     my ($scfg, $volname) = @_;
7727
7728     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7729         return $1;
7730     } else {
7731         return "raw";
7732     }
7733 }
7734
7735 sub qemu_drive_mirror {
7736     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7737
7738     $jobs = {} if !$jobs;
7739
7740     my $qemu_target;
7741     my $format;
7742     $jobs->{"drive-$drive"} = {};
7743
7744     if ($dst_volid =~ /^nbd:/) {
7745         $qemu_target = $dst_volid;
7746         $format = "nbd";
7747     } else {
7748         my $storecfg = PVE::Storage::config();
7749         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7750
7751         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7752
7753         $format = qemu_img_format($dst_scfg, $dst_volname);
7754
7755         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7756
7757         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7758     }
7759
7760     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7761     $opts->{format} = $format if $format;
7762
7763     if (defined($src_bitmap)) {
7764         $opts->{sync} = 'incremental';
7765         $opts->{bitmap} = $src_bitmap;
7766         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7767     }
7768
7769     if (defined($bwlimit)) {
7770         $opts->{speed} = $bwlimit * 1024;
7771         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7772     } else {
7773         print "drive mirror is starting for drive-$drive\n";
7774     }
7775
7776     # if a job already runs for this device we get an error, catch it for cleanup
7777     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7778     if (my $err = $@) {
7779         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7780         warn "$@\n" if $@;
7781         die "mirroring error: $err\n";
7782     }
7783
7784     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7785 }
7786
7787 # $completion can be either
7788 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7789 # 'cancel': wait until all jobs are ready, block-job-cancel them
7790 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7791 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7792 sub qemu_drive_mirror_monitor {
7793     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7794
7795     $completion //= 'complete';
7796     $op //= "mirror";
7797
7798     eval {
7799         my $err_complete = 0;
7800
7801         my $starttime = time ();
7802         while (1) {
7803             die "block job ('$op') timed out\n" if $err_complete > 300;
7804
7805             my $stats = mon_cmd($vmid, "query-block-jobs");
7806             my $ctime = time();
7807
7808             my $running_jobs = {};
7809             for my $stat (@$stats) {
7810                 next if $stat->{type} ne $op;
7811                 $running_jobs->{$stat->{device}} = $stat;
7812             }
7813
7814             my $readycounter = 0;
7815
7816             for my $job_id (sort keys %$jobs) {
7817                 my $job = $running_jobs->{$job_id};
7818
7819                 my $vanished = !defined($job);
7820                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7821                 if($complete || ($vanished && $completion eq 'auto')) {
7822                     print "$job_id: $op-job finished\n";
7823                     delete $jobs->{$job_id};
7824                     next;
7825                 }
7826
7827                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7828
7829                 my $busy = $job->{busy};
7830                 my $ready = $job->{ready};
7831                 if (my $total = $job->{len}) {
7832                     my $transferred = $job->{offset} || 0;
7833                     my $remaining = $total - $transferred;
7834                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7835
7836                     my $duration = $ctime - $starttime;
7837                     my $total_h = render_bytes($total, 1);
7838                     my $transferred_h = render_bytes($transferred, 1);
7839
7840                     my $status = sprintf(
7841                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7842                         render_duration($duration),
7843                     );
7844
7845                     if ($ready) {
7846                         if ($busy) {
7847                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7848                         } else {
7849                             $status .= ", ready";
7850                         }
7851                     }
7852                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7853                     $jobs->{$job_id}->{ready} = $ready;
7854                 }
7855
7856                 $readycounter++ if $job->{ready};
7857             }
7858
7859             last if scalar(keys %$jobs) == 0;
7860
7861             if ($readycounter == scalar(keys %$jobs)) {
7862                 print "all '$op' jobs are ready\n";
7863
7864                 # do the complete later (or has already been done)
7865                 last if $completion eq 'skip' || $completion eq 'auto';
7866
7867                 if ($vmiddst && $vmiddst != $vmid) {
7868                     my $agent_running = $qga && qga_check_running($vmid);
7869                     if ($agent_running) {
7870                         print "freeze filesystem\n";
7871                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7872                         warn $@ if $@;
7873                     } else {
7874                         print "suspend vm\n";
7875                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7876                         warn $@ if $@;
7877                     }
7878
7879                     # if we clone a disk for a new target vm, we don't switch the disk
7880                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7881
7882                     if ($agent_running) {
7883                         print "unfreeze filesystem\n";
7884                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7885                         warn $@ if $@;
7886                     } else {
7887                         print "resume vm\n";
7888                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7889                         warn $@ if $@;
7890                     }
7891
7892                     last;
7893                 } else {
7894
7895                     for my $job_id (sort keys %$jobs) {
7896                         # try to switch the disk if source and destination are on the same guest
7897                         print "$job_id: Completing block job_id...\n";
7898
7899                         my $op;
7900                         if ($completion eq 'complete') {
7901                             $op = 'block-job-complete';
7902                         } elsif ($completion eq 'cancel') {
7903                             $op = 'block-job-cancel';
7904                         } else {
7905                             die "invalid completion value: $completion\n";
7906                         }
7907                         eval { mon_cmd($vmid, $op, device => $job_id) };
7908                         if ($@ =~ m/cannot be completed/) {
7909                             print "$job_id: block job cannot be completed, trying again.\n";
7910                             $err_complete++;
7911                         }else {
7912                             print "$job_id: Completed successfully.\n";
7913                             $jobs->{$job_id}->{complete} = 1;
7914                         }
7915                     }
7916                 }
7917             }
7918             sleep 1;
7919         }
7920     };
7921     my $err = $@;
7922
7923     if ($err) {
7924         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7925         die "block job ($op) error: $err";
7926     }
7927 }
7928
7929 sub qemu_blockjobs_cancel {
7930     my ($vmid, $jobs) = @_;
7931
7932     foreach my $job (keys %$jobs) {
7933         print "$job: Cancelling block job\n";
7934         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7935         $jobs->{$job}->{cancel} = 1;
7936     }
7937
7938     while (1) {
7939         my $stats = mon_cmd($vmid, "query-block-jobs");
7940
7941         my $running_jobs = {};
7942         foreach my $stat (@$stats) {
7943             $running_jobs->{$stat->{device}} = $stat;
7944         }
7945
7946         foreach my $job (keys %$jobs) {
7947
7948             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7949                 print "$job: Done.\n";
7950                 delete $jobs->{$job};
7951             }
7952         }
7953
7954         last if scalar(keys %$jobs) == 0;
7955
7956         sleep 1;
7957     }
7958 }
7959
7960 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
7961 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
7962 my sub clone_disk_check_io_uring {
7963     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
7964
7965     return if !$use_drive_mirror;
7966
7967     # Don't complain when not changing storage.
7968     # Assume if it works for the source, it'll work for the target too.
7969     return if $src_storeid eq $dst_storeid;
7970
7971     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7972     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7973
7974     my $cache_direct = drive_uses_cache_direct($src_drive);
7975
7976     my $src_uses_io_uring;
7977     if ($src_drive->{aio}) {
7978         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
7979     } else {
7980         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
7981     }
7982
7983     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
7984         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
7985 }
7986
7987 sub clone_disk {
7988     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7989
7990     my ($vmid, $running) = $source->@{qw(vmid running)};
7991     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7992
7993     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7994     my ($storage, $format) = $dest->@{qw(storage format)};
7995
7996     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7997
7998     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7999         die "cloning from/to EFI disk requires EFI disk\n"
8000             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8001         die "cloning from/to TPM state requires TPM state\n"
8002             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8003
8004         # This would lead to two device nodes in QEMU pointing to the same backing image!
8005         die "cannot change drive name when cloning disk from/to the same VM\n"
8006             if $use_drive_mirror && $vmid == $newvmid;
8007     }
8008
8009     die "cannot move TPM state while VM is running\n"
8010         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8011
8012     my $newvolid;
8013
8014     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8015     print "$src_drivename " if $src_drivename;
8016     print "($drive->{file})\n";
8017
8018     if (!$full) {
8019         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8020         push @$newvollist, $newvolid;
8021     } else {
8022         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8023         my $storeid = $storage || $src_storeid;
8024
8025         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8026
8027         my $name = undef;
8028         my $size = undef;
8029         if (drive_is_cloudinit($drive)) {
8030             $name = "vm-$newvmid-cloudinit";
8031             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8032             if ($scfg->{path}) {
8033                 $name .= ".$dst_format";
8034             }
8035             $snapname = undef;
8036             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8037         } elsif ($dst_drivename eq 'efidisk0') {
8038             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8039         } elsif ($dst_drivename eq 'tpmstate0') {
8040             $dst_format = 'raw';
8041             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8042         } else {
8043             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8044
8045             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8046         }
8047         $newvolid = PVE::Storage::vdisk_alloc(
8048             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8049         );
8050         push @$newvollist, $newvolid;
8051
8052         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8053
8054         if (drive_is_cloudinit($drive)) {
8055             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8056             # if this is the case, we have to complete any block-jobs still there from
8057             # previous drive-mirrors
8058             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8059                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8060             }
8061             goto no_data_clone;
8062         }
8063
8064         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8065         if ($use_drive_mirror) {
8066             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8067                 $completion, $qga, $bwlimit);
8068         } else {
8069             # TODO: handle bwlimits
8070             if ($dst_drivename eq 'efidisk0') {
8071                 # the relevant data on the efidisk may be smaller than the source
8072                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8073                 # that is given by the OVMF_VARS.fd
8074                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8075                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8076
8077                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8078
8079                 # better for Ceph if block size is not too small, see bug #3324
8080                 my $bs = 1024*1024;
8081
8082                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8083
8084                 if ($src_format eq 'qcow2' && $snapname) {
8085                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8086                         if !min_version(kvm_user_version(), 6, 2);
8087                     push $cmd->@*, '-l', $snapname;
8088                 }
8089                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8090                 run_command($cmd);
8091             } else {
8092                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
8093             }
8094         }
8095     }
8096
8097 no_data_clone:
8098     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8099
8100     my $disk = dclone($drive);
8101     delete $disk->{format};
8102     $disk->{file} = $newvolid;
8103     $disk->{size} = $size if defined($size);
8104
8105     return $disk;
8106 }
8107
8108 sub get_running_qemu_version {
8109     my ($vmid) = @_;
8110     my $res = mon_cmd($vmid, "query-version");
8111     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8112 }
8113
8114 sub qemu_use_old_bios_files {
8115     my ($machine_type) = @_;
8116
8117     return if !$machine_type;
8118
8119     my $use_old_bios_files = undef;
8120
8121     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8122         $machine_type = $1;
8123         $use_old_bios_files = 1;
8124     } else {
8125         my $version = extract_version($machine_type, kvm_user_version());
8126         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8127         # load new efi bios files on migration. So this hack is required to allow
8128         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8129         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8130         $use_old_bios_files = !min_version($version, 2, 4);
8131     }
8132
8133     return ($use_old_bios_files, $machine_type);
8134 }
8135
8136 sub get_efivars_size {
8137     my ($conf, $efidisk) = @_;
8138
8139     my $arch = get_vm_arch($conf);
8140     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8141     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8142     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8143     return -s $ovmf_vars;
8144 }
8145
8146 sub update_efidisk_size {
8147     my ($conf) = @_;
8148
8149     return if !defined($conf->{efidisk0});
8150
8151     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8152     $disk->{size} = get_efivars_size($conf);
8153     $conf->{efidisk0} = print_drive($disk);
8154
8155     return;
8156 }
8157
8158 sub update_tpmstate_size {
8159     my ($conf) = @_;
8160
8161     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8162     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8163     $conf->{tpmstate0} = print_drive($disk);
8164 }
8165
8166 sub create_efidisk($$$$$$$) {
8167     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8168
8169     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8170
8171     my $vars_size_b = -s $ovmf_vars;
8172     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8173     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8174     PVE::Storage::activate_volumes($storecfg, [$volid]);
8175
8176     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8177     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8178
8179     return ($volid, $size/1024);
8180 }
8181
8182 sub vm_iothreads_list {
8183     my ($vmid) = @_;
8184
8185     my $res = mon_cmd($vmid, 'query-iothreads');
8186
8187     my $iothreads = {};
8188     foreach my $iothread (@$res) {
8189         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8190     }
8191
8192     return $iothreads;
8193 }
8194
8195 sub scsihw_infos {
8196     my ($conf, $drive) = @_;
8197
8198     my $maxdev = 0;
8199
8200     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8201         $maxdev = 7;
8202     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8203         $maxdev = 1;
8204     } else {
8205         $maxdev = 256;
8206     }
8207
8208     my $controller = int($drive->{index} / $maxdev);
8209     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8210         ? "virtioscsi"
8211         : "scsihw";
8212
8213     return ($maxdev, $controller, $controller_prefix);
8214 }
8215
8216 sub resolve_dst_disk_format {
8217         my ($storecfg, $storeid, $src_volname, $format) = @_;
8218         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8219
8220         if (!$format) {
8221             # if no target format is specified, use the source disk format as hint
8222             if ($src_volname) {
8223                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8224                 $format = qemu_img_format($scfg, $src_volname);
8225             } else {
8226                 return $defFormat;
8227             }
8228         }
8229
8230         # test if requested format is supported - else use default
8231         my $supported = grep { $_ eq $format } @$validFormats;
8232         $format = $defFormat if !$supported;
8233         return $format;
8234 }
8235
8236 # NOTE: if this logic changes, please update docs & possibly gui logic
8237 sub find_vmstate_storage {
8238     my ($conf, $storecfg) = @_;
8239
8240     # first, return storage from conf if set
8241     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8242
8243     my ($target, $shared, $local);
8244
8245     foreach_storage_used_by_vm($conf, sub {
8246         my ($sid) = @_;
8247         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8248         my $dst = $scfg->{shared} ? \$shared : \$local;
8249         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8250     });
8251
8252     # second, use shared storage where VM has at least one disk
8253     # third, use local storage where VM has at least one disk
8254     # fall back to local storage
8255     $target = $shared // $local // 'local';
8256
8257     return $target;
8258 }
8259
8260 sub generate_uuid {
8261     my ($uuid, $uuid_str);
8262     UUID::generate($uuid);
8263     UUID::unparse($uuid, $uuid_str);
8264     return $uuid_str;
8265 }
8266
8267 sub generate_smbios1_uuid {
8268     return "uuid=".generate_uuid();
8269 }
8270
8271 sub nbd_stop {
8272     my ($vmid) = @_;
8273
8274     mon_cmd($vmid, 'nbd-server-stop');
8275 }
8276
8277 sub create_reboot_request {
8278     my ($vmid) = @_;
8279     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8280         or die "failed to create reboot trigger file: $!\n";
8281     close($fh);
8282 }
8283
8284 sub clear_reboot_request {
8285     my ($vmid) = @_;
8286     my $path = "/run/qemu-server/$vmid.reboot";
8287     my $res = 0;
8288
8289     $res = unlink($path);
8290     die "could not remove reboot request for $vmid: $!"
8291         if !$res && $! != POSIX::ENOENT;
8292
8293     return $res;
8294 }
8295
8296 sub bootorder_from_legacy {
8297     my ($conf, $bootcfg) = @_;
8298
8299     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8300     my $bootindex_hash = {};
8301     my $i = 1;
8302     foreach my $o (split(//, $boot)) {
8303         $bootindex_hash->{$o} = $i*100;
8304         $i++;
8305     }
8306
8307     my $bootorder = {};
8308
8309     PVE::QemuConfig->foreach_volume($conf, sub {
8310         my ($ds, $drive) = @_;
8311
8312         if (drive_is_cdrom ($drive, 1)) {
8313             if ($bootindex_hash->{d}) {
8314                 $bootorder->{$ds} = $bootindex_hash->{d};
8315                 $bootindex_hash->{d} += 1;
8316             }
8317         } elsif ($bootindex_hash->{c}) {
8318             $bootorder->{$ds} = $bootindex_hash->{c}
8319                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8320             $bootindex_hash->{c} += 1;
8321         }
8322     });
8323
8324     if ($bootindex_hash->{n}) {
8325         for (my $i = 0; $i < $MAX_NETS; $i++) {
8326             my $netname = "net$i";
8327             next if !$conf->{$netname};
8328             $bootorder->{$netname} = $bootindex_hash->{n};
8329             $bootindex_hash->{n} += 1;
8330         }
8331     }
8332
8333     return $bootorder;
8334 }
8335
8336 # Generate default device list for 'boot: order=' property. Matches legacy
8337 # default boot order, but with explicit device names. This is important, since
8338 # the fallback for when neither 'order' nor the old format is specified relies
8339 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8340 sub get_default_bootdevices {
8341     my ($conf) = @_;
8342
8343     my @ret = ();
8344
8345     # harddisk
8346     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8347     push @ret, $first if $first;
8348
8349     # cdrom
8350     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8351     push @ret, $first if $first;
8352
8353     # network
8354     for (my $i = 0; $i < $MAX_NETS; $i++) {
8355         my $netname = "net$i";
8356         next if !$conf->{$netname};
8357         push @ret, $netname;
8358         last;
8359     }
8360
8361     return \@ret;
8362 }
8363
8364 sub device_bootorder {
8365     my ($conf) = @_;
8366
8367     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8368
8369     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8370
8371     my $bootorder = {};
8372     if (!defined($boot) || $boot->{legacy}) {
8373         $bootorder = bootorder_from_legacy($conf, $boot);
8374     } elsif ($boot->{order}) {
8375         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8376         for my $dev (PVE::Tools::split_list($boot->{order})) {
8377             $bootorder->{$dev} = $i++;
8378         }
8379     }
8380
8381     return $bootorder;
8382 }
8383
8384 sub register_qmeventd_handle {
8385     my ($vmid) = @_;
8386
8387     my $fh;
8388     my $peer = "/var/run/qmeventd.sock";
8389     my $count = 0;
8390
8391     for (;;) {
8392         $count++;
8393         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8394         last if $fh;
8395         if ($! != EINTR && $! != EAGAIN) {
8396             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8397         }
8398         if ($count > 4) {
8399             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8400               . "after $count retries\n";
8401         }
8402         usleep(25000);
8403     }
8404
8405     # send handshake to mark VM as backing up
8406     print $fh to_json({vzdump => {vmid => "$vmid"}});
8407
8408     # return handle to be closed later when inhibit is no longer required
8409     return $fh;
8410 }
8411
8412 # bash completion helper
8413
8414 sub complete_backup_archives {
8415     my ($cmdname, $pname, $cvalue) = @_;
8416
8417     my $cfg = PVE::Storage::config();
8418
8419     my $storeid;
8420
8421     if ($cvalue =~ m/^([^:]+):/) {
8422         $storeid = $1;
8423     }
8424
8425     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8426
8427     my $res = [];
8428     foreach my $id (keys %$data) {
8429         foreach my $item (@{$data->{$id}}) {
8430             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8431             push @$res, $item->{volid} if defined($item->{volid});
8432         }
8433     }
8434
8435     return $res;
8436 }
8437
8438 my $complete_vmid_full = sub {
8439     my ($running) = @_;
8440
8441     my $idlist = vmstatus();
8442
8443     my $res = [];
8444
8445     foreach my $id (keys %$idlist) {
8446         my $d = $idlist->{$id};
8447         if (defined($running)) {
8448             next if $d->{template};
8449             next if $running && $d->{status} ne 'running';
8450             next if !$running && $d->{status} eq 'running';
8451         }
8452         push @$res, $id;
8453
8454     }
8455     return $res;
8456 };
8457
8458 sub complete_vmid {
8459     return &$complete_vmid_full();
8460 }
8461
8462 sub complete_vmid_stopped {
8463     return &$complete_vmid_full(0);
8464 }
8465
8466 sub complete_vmid_running {
8467     return &$complete_vmid_full(1);
8468 }
8469
8470 sub complete_storage {
8471
8472     my $cfg = PVE::Storage::config();
8473     my $ids = $cfg->{ids};
8474
8475     my $res = [];
8476     foreach my $sid (keys %$ids) {
8477         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8478         next if !$ids->{$sid}->{content}->{images};
8479         push @$res, $sid;
8480     }
8481
8482     return $res;
8483 }
8484
8485 sub complete_migration_storage {
8486     my ($cmd, $param, $current_value, $all_args) = @_;
8487
8488     my $targetnode = @$all_args[1];
8489
8490     my $cfg = PVE::Storage::config();
8491     my $ids = $cfg->{ids};
8492
8493     my $res = [];
8494     foreach my $sid (keys %$ids) {
8495         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8496         next if !$ids->{$sid}->{content}->{images};
8497         push @$res, $sid;
8498     }
8499
8500     return $res;
8501 }
8502
8503 sub vm_is_paused {
8504     my ($vmid) = @_;
8505     my $qmpstatus = eval {
8506         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8507         mon_cmd($vmid, "query-status");
8508     };
8509     warn "$@\n" if $@;
8510     return $qmpstatus && $qmpstatus->{status} eq "paused";
8511 }
8512
8513 sub check_volume_storage_type {
8514     my ($storecfg, $vol) = @_;
8515
8516     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8517     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8518     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8519
8520     die "storage '$storeid' does not support content-type '$vtype'\n"
8521         if !$scfg->{content}->{$vtype};
8522
8523     return 1;
8524 }
8525
8526 sub add_nets_bridge_fdb {
8527     my ($conf, $vmid) = @_;
8528
8529     for my $opt (keys %$conf) {
8530         next if $opt !~ m/^net(\d+)$/;
8531         my $iface = "tap${vmid}i$1";
8532         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8533         my $net = parse_net($conf->{$opt}, 1) or next;
8534
8535         my $mac = $net->{macaddr};
8536         if (!$mac) {
8537             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8538                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8539             next;
8540         }
8541
8542         my $bridge = $net->{bridge};
8543         if ($have_sdn) {
8544             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8545         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8546             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8547         }
8548     }
8549 }
8550
8551 sub del_nets_bridge_fdb {
8552     my ($conf, $vmid) = @_;
8553
8554     for my $opt (keys %$conf) {
8555         next if $opt !~ m/^net(\d+)$/;
8556         my $iface = "tap${vmid}i$1";
8557
8558         my $net = parse_net($conf->{$opt}) or next;
8559         my $mac = $net->{macaddr} or next;
8560
8561         my $bridge = $net->{bridge};
8562         if ($have_sdn) {
8563             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8564         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8565             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8566         }
8567     }
8568 }
8569
8570 1;