PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::INotify;
  38 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  39 use PVE::ProcFSTools;
  40 use PVE::PBSClient;
  41 use PVE::RESTEnvironment qw(log_warn);
  42 use PVE::RPCEnvironment;
  43 use PVE::Storage;
  44 use PVE::SysFSTools;
  45 use PVE::Systemd;
  46 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  47
  48 use PVE::QMPClient;
  49 use PVE::QemuConfig;
  50 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  51 use PVE::QemuServer::Cloudinit;
  52 use PVE::QemuServer::CGroup;
  53 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  54 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  55 use PVE::QemuServer::Machine;
  56 use PVE::QemuServer::Memory;
  57 use PVE::QemuServer::Monitor qw(mon_cmd);
  58 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  59 use PVE::QemuServer::USB qw(parse_usb_device);
  60
  61 my $have_sdn;
  62 eval {
  63     require PVE::Network::SDN::Zones;
  64     $have_sdn = 1;
  65 };
  66
  67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  68 my $OVMF = {
  69     x86_64 => {
  70         '4m-no-smm' => [
  71             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  72             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  73         ],
  74         '4m-no-smm-ms' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  77         ],
  78         '4m' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  81         ],
  82         '4m-ms' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  85         ],
  86         default => [
  87             "$EDK2_FW_BASE/OVMF_CODE.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS.fd",
  89         ],
  90     },
  91     aarch64 => {
  92         default => [
  93             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  94             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  95         ],
  96     },
  97 };
  98
  99 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 100
 101 # Note about locking: we use flock on the config file protect against concurent actions.
 102 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 104 # But you can ignore this kind of lock with the --skiplock flag.
 105
 106 cfs_register_file(
 107     '/qemu-server/',
 108     \&parse_vm_config,
 109     \&write_vm_config
 110 );
 111
 112 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 113     description => "Some command save/restore state from this location.",
 114     type => 'string',
 115     maxLength => 128,
 116     optional => 1,
 117 });
 118
 119 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 120         description => "Specifies the QEMU machine type.",
 121         type => 'string',
 122         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 123         maxLength => 40,
 124         optional => 1,
 125 });
 126
 127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 244             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 245             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 246             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 247             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 248             ." a hardware RNG from the host.",
 249     },
 250     max_bytes => {
 251         type => 'integer',
 252         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 253             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 254             ." `0` to disable limiting (potentially dangerous!).",
 255         optional => 1,
 256
 257         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 258         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 259         # reading from /dev/urandom
 260         default => 1024,
 261     },
 262     period => {
 263         type => 'integer',
 264         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 265             ." the guest to retrieve another 'max_bytes' of entropy.",
 266         optional => 1,
 267         default => 1000,
 268     },
 269 };
 270
 271 my $meta_info_fmt = {
 272     'ctime' => {
 273         type => 'integer',
 274         description => "The guest creation timestamp as UNIX epoch time",
 275         minimum => 0,
 276         optional => 1,
 277     },
 278     'creation-qemu' => {
 279         type => 'string',
 280         description => "The QEMU (machine) version from the time this VM was created.",
 281         pattern => '\d+(\.\d+)+',
 282         optional => 1,
 283     },
 284 };
 285
 286 my $confdesc = {
 287     onboot => {
 288         optional => 1,
 289         type => 'boolean',
 290         description => "Specifies whether a VM will be started during system bootup.",
 291         default => 0,
 292     },
 293     autostart => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Automatic restart after crash (currently ignored).",
 297         default => 0,
 298     },
 299     hotplug => {
 300         optional => 1,
 301         type => 'string', format => 'pve-hotplug-features',
 302         description => "Selectively enable hotplug features. This is a comma separated list of"
 303             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 304             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 305             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 306             ." windows > 7.",
 307         default => 'network,disk,usb',
 308     },
 309     reboot => {
 310         optional => 1,
 311         type => 'boolean',
 312         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 313         default => 1,
 314     },
 315     lock => {
 316         optional => 1,
 317         type => 'string',
 318         description => "Lock/unlock the VM.",
 319         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 320     },
 321     cpulimit => {
 322         optional => 1,
 323         type => 'number',
 324         description => "Limit of CPU usage.",
 325         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 326             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 327         minimum => 0,
 328         maximum => 128,
 329         default => 0,
 330     },
 331     cpuunits => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 335         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 336             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 337             ." weights of all the other running VMs.",
 338         minimum => 1,
 339         maximum => 262144,
 340         default => 'cgroup v1: 1024, cgroup v2: 100',
 341     },
 342     memory => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 346             ." you use the balloon device.",
 347         minimum => 16,
 348         default => 512,
 349     },
 350     balloon => {
 351         optional => 1,
 352         type => 'integer',
 353         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 354         minimum => 0,
 355     },
 356     shares => {
 357         optional => 1,
 358         type => 'integer',
 359         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 360             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 361             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 362         minimum => 0,
 363         maximum => 50000,
 364         default => 1000,
 365     },
 366     keyboard => {
 367         optional => 1,
 368         type => 'string',
 369         description => "Keyboard layout for VNC server. This option is generally not required and"
 370         ." is often better handled from within the guest OS.",
 371         enum => PVE::Tools::kvmkeymaplist(),
 372         default => undef,
 373     },
 374     name => {
 375         optional => 1,
 376         type => 'string', format => 'dns-name',
 377         description => "Set a name for the VM. Only used on the configuration web interface.",
 378     },
 379     scsihw => {
 380         optional => 1,
 381         type => 'string',
 382         description => "SCSI controller model",
 383         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 384         default => 'lsi',
 385     },
 386     description => {
 387         optional => 1,
 388         type => 'string',
 389         description => "Description for the VM. Shown in the web-interface VM's summary."
 390             ." This is saved as comment inside the configuration file.",
 391         maxLength => 1024 * 8,
 392     },
 393     ostype => {
 394         optional => 1,
 395         type => 'string',
 396         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 397         description => "Specify guest operating system.",
 398         verbose_description => <<EODESC,
 399 Specify guest operating system. This is used to enable special
 400 optimization/features for specific operating systems:
 401
 402 [horizontal]
 403 other;; unspecified OS
 404 wxp;; Microsoft Windows XP
 405 w2k;; Microsoft Windows 2000
 406 w2k3;; Microsoft Windows 2003
 407 w2k8;; Microsoft Windows 2008
 408 wvista;; Microsoft Windows Vista
 409 win7;; Microsoft Windows 7
 410 win8;; Microsoft Windows 8/2012/2012r2
 411 win10;; Microsoft Windows 10/2016/2019
 412 win11;; Microsoft Windows 11/2022
 413 l24;; Linux 2.4 Kernel
 414 l26;; Linux 2.6 - 6.X Kernel
 415 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 416 EODESC
 417     },
 418     boot => {
 419         optional => 1,
 420         type => 'string', format => 'pve-qm-boot',
 421         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 422             ." key or 'legacy=' is deprecated.",
 423     },
 424     bootdisk => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-bootdisk',
 427         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 428         pattern => '(ide|sata|scsi|virtio)\d+',
 429     },
 430     smp => {
 431         optional => 1,
 432         type => 'integer',
 433         description => "The number of CPUs. Please use option -sockets instead.",
 434         minimum => 1,
 435         default => 1,
 436     },
 437     sockets => {
 438         optional => 1,
 439         type => 'integer',
 440         description => "The number of CPU sockets.",
 441         minimum => 1,
 442         default => 1,
 443     },
 444     cores => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "The number of cores per socket.",
 448         minimum => 1,
 449         default => 1,
 450     },
 451     numa => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable NUMA.",
 455         default => 0,
 456     },
 457     hugepages => {
 458         optional => 1,
 459         type => 'string',
 460         description => "Enable/disable hugepages memory.",
 461         enum => [qw(any 2 1024)],
 462     },
 463     keephugepages => {
 464         optional => 1,
 465         type => 'boolean',
 466         default => 0,
 467         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 468             ." after VM shutdown and can be used for subsequent starts.",
 469     },
 470     vcpus => {
 471         optional => 1,
 472         type => 'integer',
 473         description => "Number of hotplugged vcpus.",
 474         minimum => 1,
 475         default => 0,
 476     },
 477     acpi => {
 478         optional => 1,
 479         type => 'boolean',
 480         description => "Enable/disable ACPI.",
 481         default => 1,
 482     },
 483     agent => {
 484         optional => 1,
 485         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 486         type => 'string',
 487         format => $agent_fmt,
 488     },
 489     kvm => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Enable/disable KVM hardware virtualization.",
 493         default => 1,
 494     },
 495     tdf => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable time drift fix.",
 499         default => 0,
 500     },
 501     localtime => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 505             ." the `ostype` indicates a Microsoft Windows OS.",
 506     },
 507     freeze => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 511     },
 512     vga => {
 513         optional => 1,
 514         type => 'string', format => $vga_fmt,
 515         description => "Configure the VGA hardware.",
 516         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 517             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 518             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 519             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 520             ." display server. For win* OS you can select how many independent displays you want,"
 521             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 522             ." using a serial device as terminal.",
 523     },
 524     watchdog => {
 525         optional => 1,
 526         type => 'string', format => 'pve-qm-watchdog',
 527         description => "Create a virtual hardware watchdog device.",
 528         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 529             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 530             ." else the watchdog will reset the guest (or execute the respective action specified)",
 531     },
 532     startdate => {
 533         optional => 1,
 534         type => 'string',
 535         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 536         description => "Set the initial date of the real time clock. Valid format for date are:"
 537             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 538         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 539         default => 'now',
 540     },
 541     startup =>  get_standard_option('pve-startup-order'),
 542     template => {
 543         optional => 1,
 544         type => 'boolean',
 545         description => "Enable/disable Template.",
 546         default => 0,
 547     },
 548     args => {
 549         optional => 1,
 550         type => 'string',
 551         description => "Arbitrary arguments passed to kvm.",
 552         verbose_description => <<EODESCR,
 553 Arbitrary arguments passed to kvm, for example:
 554
 555 args: -no-reboot -no-hpet
 556
 557 NOTE: this option is for experts only.
 558 EODESCR
 559     },
 560     tablet => {
 561         optional => 1,
 562         type => 'boolean',
 563         default => 1,
 564         description => "Enable/disable the USB tablet device.",
 565         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 566             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 567             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 568             ." may consider disabling this to save some context switches. This is turned off by"
 569             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 570     },
 571     migrate_speed => {
 572         optional => 1,
 573         type => 'integer',
 574         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 575         minimum => 0,
 576         default => 0,
 577     },
 578     migrate_downtime => {
 579         optional => 1,
 580         type => 'number',
 581         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 582         minimum => 0,
 583         default => 0.1,
 584     },
 585     cdrom => {
 586         optional => 1,
 587         type => 'string', format => 'pve-qm-ide',
 588         typetext => '<volume>',
 589         description => "This is an alias for option -ide2",
 590     },
 591     cpu => {
 592         optional => 1,
 593         description => "Emulated CPU type.",
 594         type => 'string',
 595         format => 'pve-vm-cpu-conf',
 596     },
 597     parent => get_standard_option('pve-snapshot-name', {
 598         optional => 1,
 599         description => "Parent snapshot name. This is used internally, and should not be modified.",
 600     }),
 601     snaptime => {
 602         optional => 1,
 603         description => "Timestamp for snapshots.",
 604         type => 'integer',
 605         minimum => 0,
 606     },
 607     vmstate => {
 608         optional => 1,
 609         type => 'string', format => 'pve-volume-id',
 610         description => "Reference to a volume which stores the VM state. This is used internally"
 611             ." for snapshots.",
 612     },
 613     vmstatestorage => get_standard_option('pve-storage-id', {
 614         description => "Default storage for VM state volumes/files.",
 615         optional => 1,
 616     }),
 617     runningmachine => get_standard_option('pve-qemu-machine', {
 618         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 619             ." for snapshots.",
 620     }),
 621     runningcpu => {
 622         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 623             ." internally for snapshots.",
 624         optional => 1,
 625         type => 'string',
 626         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 627         format_description => 'QEMU -cpu parameter'
 628     },
 629     machine => get_standard_option('pve-qemu-machine'),
 630     arch => {
 631         description => "Virtual processor architecture. Defaults to the host.",
 632         optional => 1,
 633         type => 'string',
 634         enum => [qw(x86_64 aarch64)],
 635     },
 636     smbios1 => {
 637         description => "Specify SMBIOS type 1 fields.",
 638         type => 'string', format => 'pve-qm-smbios1',
 639         maxLength => 512,
 640         optional => 1,
 641     },
 642     protection => {
 643         optional => 1,
 644         type => 'boolean',
 645         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 646             ." remove disk operations.",
 647         default => 0,
 648     },
 649     bios => {
 650         optional => 1,
 651         type => 'string',
 652         enum => [ qw(seabios ovmf) ],
 653         description => "Select BIOS implementation.",
 654         default => 'seabios',
 655     },
 656     vmgenid => {
 657         type => 'string',
 658         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 659         format_description => 'UUID',
 660         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 661             ." to disable explicitly.",
 662         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 663             ." value identifier to the guest OS. This allows to notify the guest operating system"
 664             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 665             ." execution or creation from a template). The guest operating system notices the"
 666             ." change, and is then able to react as appropriate by marking its copies of"
 667             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 668             ."Note that auto-creation only works when done through API/CLI create or update methods"
 669             .", but not when manually editing the config file.",
 670         default => "1 (autogenerated)",
 671         optional => 1,
 672     },
 673     hookscript => {
 674         type => 'string',
 675         format => 'pve-volume-id',
 676         optional => 1,
 677         description => "Script that will be executed during various steps in the vms lifetime.",
 678     },
 679     ivshmem => {
 680         type => 'string',
 681         format => $ivshmem_fmt,
 682         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 683             ." the host.",
 684         optional => 1,
 685     },
 686     audio0 => {
 687         type => 'string',
 688         format => $audio_fmt,
 689         description => "Configure a audio device, useful in combination with QXL/Spice.",
 690         optional => 1
 691     },
 692     spice_enhancements => {
 693         type => 'string',
 694         format => $spice_enhancements_fmt,
 695         description => "Configure additional enhancements for SPICE.",
 696         optional => 1
 697     },
 698     tags => {
 699         type => 'string', format => 'pve-tag-list',
 700         description => 'Tags of the VM. This is only meta information.',
 701         optional => 1,
 702     },
 703     rng0 => {
 704         type => 'string',
 705         format => $rng_fmt,
 706         description => "Configure a VirtIO-based Random Number Generator.",
 707         optional => 1,
 708     },
 709     meta => {
 710         type => 'string',
 711         format => $meta_info_fmt,
 712         description => "Some (read-only) meta-information about this guest.",
 713         optional => 1,
 714     },
 715     affinity => {
 716         type => 'string', format => 'pve-cpuset',
 717         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 718         optional => 1,
 719     },
 720 };
 721
 722 my $cicustom_fmt = {
 723     meta => {
 724         type => 'string',
 725         optional => 1,
 726         description => 'Specify a custom file containing all meta data passed to the VM via"
 727             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731     network => {
 732         type => 'string',
 733         optional => 1,
 734         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 735         format => 'pve-volume-id',
 736         format_description => 'volume',
 737     },
 738     user => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 742         format => 'pve-volume-id',
 743         format_description => 'volume',
 744     },
 745     vendor => {
 746         type => 'string',
 747         optional => 1,
 748         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 749         format => 'pve-volume-id',
 750         format_description => 'volume',
 751     },
 752 };
 753 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 754
 755 my $confdesc_cloudinit = {
 756     citype => {
 757         optional => 1,
 758         type => 'string',
 759         description => 'Specifies the cloud-init configuration format. The default depends on the'
 760             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 761             .' and `configdrive2` for windows.',
 762         enum => ['configdrive2', 'nocloud', 'opennebula'],
 763     },
 764     ciuser => {
 765         optional => 1,
 766         type => 'string',
 767         description => "cloud-init: User name to change ssh keys and password for instead of the"
 768             ." image's configured default user.",
 769     },
 770     cipassword => {
 771         optional => 1,
 772         type => 'string',
 773         description => 'cloud-init: Password to assign the user. Using this is generally not'
 774             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 775             .' support hashed passwords.',
 776     },
 777     cicustom => {
 778         optional => 1,
 779         type => 'string',
 780         description => 'cloud-init: Specify custom files to replace the automatically generated'
 781             .' ones at start.',
 782         format => 'pve-qm-cicustom',
 783     },
 784     searchdomain => {
 785         optional => 1,
 786         type => 'string',
 787         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 788             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 789             .' are set.',
 790     },
 791     nameserver => {
 792         optional => 1,
 793         type => 'string', format => 'address-list',
 794         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 795             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 796             .' are set.',
 797     },
 798     sshkeys => {
 799         optional => 1,
 800         type => 'string',
 801         format => 'urlencoded',
 802         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 803     },
 804 };
 805
 806 # what about other qemu settings ?
 807 #cpu => 'string',
 808 #machine => 'string',
 809 #fda => 'file',
 810 #fdb => 'file',
 811 #mtdblock => 'file',
 812 #sd => 'file',
 813 #pflash => 'file',
 814 #snapshot => 'bool',
 815 #bootp => 'file',
 816 ##tftp => 'dir',
 817 ##smb => 'dir',
 818 #kernel => 'file',
 819 #append => 'string',
 820 #initrd => 'file',
 821 ##soundhw => 'string',
 822
 823 while (my ($k, $v) = each %$confdesc) {
 824     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 825 }
 826
 827 my $MAX_USB_DEVICES = 14;
 828 my $MAX_NETS = 32;
 829 my $MAX_SERIAL_PORTS = 4;
 830 my $MAX_PARALLEL_PORTS = 3;
 831 my $MAX_NUMA = 8;
 832
 833 my $numa_fmt = {
 834     cpus => {
 835         type => "string",
 836         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 837         description => "CPUs accessing this NUMA node.",
 838         format_description => "id[-id];...",
 839     },
 840     memory => {
 841         type => "number",
 842         description => "Amount of memory this NUMA node provides.",
 843         optional => 1,
 844     },
 845     hostnodes => {
 846         type => "string",
 847         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 848         description => "Host NUMA nodes to use.",
 849         format_description => "id[-id];...",
 850         optional => 1,
 851     },
 852     policy => {
 853         type => 'string',
 854         enum => [qw(preferred bind interleave)],
 855         description => "NUMA allocation policy.",
 856         optional => 1,
 857     },
 858 };
 859 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 860 my $numadesc = {
 861     optional => 1,
 862     type => 'string', format => $numa_fmt,
 863     description => "NUMA topology.",
 864 };
 865 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 866
 867 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 868     $confdesc->{"numa$i"} = $numadesc;
 869 }
 870
 871 my $nic_model_list = [
 872     'e1000',
 873     'e1000-82540em',
 874     'e1000-82544gc',
 875     'e1000-82545em',
 876     'e1000e',
 877     'i82551',
 878     'i82557b',
 879     'i82559er',
 880     'ne2k_isa',
 881     'ne2k_pci',
 882     'pcnet',
 883     'rtl8139',
 884     'virtio',
 885     'vmxnet3',
 886 ];
 887 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 888
 889 my $net_fmt_bridge_descr = <<__EOD__;
 890 Bridge to attach the network device to. The Proxmox VE standard bridge
 891 is called 'vmbr0'.
 892
 893 If you do not specify a bridge, we create a kvm user (NATed) network
 894 device, which provides DHCP and DNS services. The following addresses
 895 are used:
 896
 897  10.0.2.2   Gateway
 898  10.0.2.3   DNS Server
 899  10.0.2.4   SMB Server
 900
 901 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 902 __EOD__
 903
 904 my $net_fmt = {
 905     macaddr  => get_standard_option('mac-addr', {
 906         description => "MAC address. That address must be unique withing your network. This is"
 907             ." automatically generated if not specified.",
 908     }),
 909     model => {
 910         type => 'string',
 911         description => "Network Card Model. The 'virtio' model provides the best performance with"
 912             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 913             ." best to use 'e1000'.",
 914         enum => $nic_model_list,
 915         default_key => 1,
 916     },
 917     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 918     bridge => get_standard_option('pve-bridge-id', {
 919         description => $net_fmt_bridge_descr,
 920         optional => 1,
 921     }),
 922     queues => {
 923         type => 'integer',
 924         minimum => 0, maximum => 64,
 925         description => 'Number of packet queues to be used on the device.',
 926         optional => 1,
 927     },
 928     rate => {
 929         type => 'number',
 930         minimum => 0,
 931         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 932         optional => 1,
 933     },
 934     tag => {
 935         type => 'integer',
 936         minimum => 1, maximum => 4094,
 937         description => 'VLAN tag to apply to packets on this interface.',
 938         optional => 1,
 939     },
 940     trunks => {
 941         type => 'string',
 942         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 943         description => 'VLAN trunks to pass through this interface.',
 944         format_description => 'vlanid[;vlanid...]',
 945         optional => 1,
 946     },
 947     firewall => {
 948         type => 'boolean',
 949         description => 'Whether this interface should be protected by the firewall.',
 950         optional => 1,
 951     },
 952     link_down => {
 953         type => 'boolean',
 954         description => 'Whether this interface should be disconnected (like pulling the plug).',
 955         optional => 1,
 956     },
 957     mtu => {
 958         type => 'integer',
 959         minimum => 1, maximum => 65520,
 960         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 961         optional => 1,
 962     },
 963 };
 964
 965 my $netdesc = {
 966     optional => 1,
 967     type => 'string', format => $net_fmt,
 968     description => "Specify network devices.",
 969 };
 970
 971 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 972
 973 my $ipconfig_fmt = {
 974     ip => {
 975         type => 'string',
 976         format => 'pve-ipv4-config',
 977         format_description => 'IPv4Format/CIDR',
 978         description => 'IPv4 address in CIDR format.',
 979         optional => 1,
 980         default => 'dhcp',
 981     },
 982     gw => {
 983         type => 'string',
 984         format => 'ipv4',
 985         format_description => 'GatewayIPv4',
 986         description => 'Default gateway for IPv4 traffic.',
 987         optional => 1,
 988         requires => 'ip',
 989     },
 990     ip6 => {
 991         type => 'string',
 992         format => 'pve-ipv6-config',
 993         format_description => 'IPv6Format/CIDR',
 994         description => 'IPv6 address in CIDR format.',
 995         optional => 1,
 996         default => 'dhcp',
 997     },
 998     gw6 => {
 999         type => 'string',
1000         format => 'ipv6',
1001         format_description => 'GatewayIPv6',
1002         description => 'Default gateway for IPv6 traffic.',
1003         optional => 1,
1004         requires => 'ip6',
1005     },
1006 };
1007 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1008 my $ipconfigdesc = {
1009     optional => 1,
1010     type => 'string', format => 'pve-qm-ipconfig',
1011     description => <<'EODESCR',
1012 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1013
1014 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1015
1016 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1017 gateway should be provided.
1018 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1019 cloud-init 19.4 or newer.
1020
1021 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1022 dhcp on IPv4.
1023 EODESCR
1024 };
1025 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1026
1027 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1028     $confdesc->{"net$i"} = $netdesc;
1029     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1030 }
1031
1032 foreach my $key (keys %$confdesc_cloudinit) {
1033     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1034 }
1035
1036 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1037 sub pve_verify_cpuset {
1038     my ($set_text, $noerr) = @_;
1039
1040     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1041
1042     if ($@) {
1043         return if $noerr;
1044         die "unable to parse cpuset option\n";
1045     }
1046
1047     return PVE::CpuSet->new($members)->short_string();
1048 }
1049
1050 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1051 sub verify_volume_id_or_qm_path {
1052     my ($volid, $noerr) = @_;
1053
1054     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1055
1056     return verify_volume_id_or_absolute_path($volid, $noerr);
1057 }
1058
1059 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1060 sub verify_volume_id_or_absolute_path {
1061     my ($volid, $noerr) = @_;
1062
1063     return $volid if $volid =~ m|^/|;
1064
1065     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1066     if ($@) {
1067         return if $noerr;
1068         die $@;
1069     }
1070     return $volid;
1071 }
1072
1073 my $usb_fmt = {
1074     host => {
1075         default_key => 1,
1076         type => 'string', format => 'pve-qm-usb-device',
1077         format_description => 'HOSTUSBDEVICE|spice',
1078         description => <<EODESCR,
1079 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1080
1081  'bus-port(.port)*' (decimal numbers) or
1082  'vendor_id:product_id' (hexadeciaml numbers) or
1083  'spice'
1084
1085 You can use the 'lsusb -t' command to list existing usb devices.
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1089
1090 The value 'spice' can be used to add a usb redirection devices for spice.
1091 EODESCR
1092     },
1093     usb3 => {
1094         optional => 1,
1095         type => 'boolean',
1096         description => "Specifies whether if given host option is a USB3 device or port."
1097             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1098             ." is irrelevant (all devices are plugged into a xhci controller).",
1099         default => 0,
1100     },
1101 };
1102
1103 my $usbdesc = {
1104     optional => 1,
1105     type => 'string', format => $usb_fmt,
1106     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1107         ." l26 or windows > 7, n can be up to 14).",
1108 };
1109 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1110
1111 my $serialdesc = {
1112         optional => 1,
1113         type => 'string',
1114         pattern => '(/dev/.+|socket)',
1115         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1116         verbose_description =>  <<EODESCR,
1117 Create a serial device inside the VM (n is 0 to 3), and pass through a
1118 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1119 host side (use 'qm terminal' to open a terminal connection).
1120
1121 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1122 use with special care.
1123
1124 CAUTION: Experimental! User reported problems with this option.
1125 EODESCR
1126 };
1127
1128 my $paralleldesc= {
1129         optional => 1,
1130         type => 'string',
1131         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1132         description =>  "Map host parallel devices (n is 0 to 2).",
1133         verbose_description =>  <<EODESCR,
1134 Map host parallel devices (n is 0 to 2).
1135
1136 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1137 machines - use with special care.
1138
1139 CAUTION: Experimental! User reported problems with this option.
1140 EODESCR
1141 };
1142
1143 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1144     $confdesc->{"parallel$i"} = $paralleldesc;
1145 }
1146
1147 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1148     $confdesc->{"serial$i"} = $serialdesc;
1149 }
1150
1151 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1152     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1153 }
1154
1155 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1156     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1157 }
1158
1159 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1160     $confdesc->{"usb$i"} = $usbdesc;
1161 }
1162
1163 my $boot_fmt = {
1164     legacy => {
1165         optional => 1,
1166         default_key => 1,
1167         type => 'string',
1168         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1169                      . " Deprecated, use 'order=' instead.",
1170         pattern => '[acdn]{1,4}',
1171         format_description => "[acdn]{1,4}",
1172
1173         # note: this is also the fallback if boot: is not given at all
1174         default => 'cdn',
1175     },
1176     order => {
1177         optional => 1,
1178         type => 'string',
1179         format => 'pve-qm-bootdev-list',
1180         format_description => "device[;device...]",
1181         description => <<EODESC,
1182 The guest will attempt to boot from devices in the order they appear here.
1183
1184 Disks, optical drives and passed-through storage USB devices will be directly
1185 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1186 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1187
1188 Note that only devices in this list will be marked as bootable and thus loaded
1189 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1190 (e.g. software-raid), you need to specify all of them here.
1191
1192 Overrides the deprecated 'legacy=[acdn]*' value when given.
1193 EODESC
1194     },
1195 };
1196 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1197
1198 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1199 sub verify_bootdev {
1200     my ($dev, $noerr) = @_;
1201
1202     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1203     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1204
1205     my $check = sub {
1206         my ($base) = @_;
1207         return 0 if $dev !~ m/^$base\d+$/;
1208         return 0 if !$confdesc->{$dev};
1209         return 1;
1210     };
1211
1212     return $dev if $check->("net");
1213     return $dev if $check->("usb");
1214     return $dev if $check->("hostpci");
1215
1216     return if $noerr;
1217     die "invalid boot device '$dev'\n";
1218 }
1219
1220 sub print_bootorder {
1221     my ($devs) = @_;
1222     return "" if !@$devs;
1223     my $data = { order => join(';', @$devs) };
1224     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1225 }
1226
1227 my $kvm_api_version = 0;
1228
1229 sub kvm_version {
1230     return $kvm_api_version if $kvm_api_version;
1231
1232     open my $fh, '<', '/dev/kvm' or return;
1233
1234     # 0xae00 => KVM_GET_API_VERSION
1235     $kvm_api_version = ioctl($fh, 0xae00, 0);
1236     close($fh);
1237
1238     return $kvm_api_version;
1239 }
1240
1241 my $kvm_user_version = {};
1242 my $kvm_mtime = {};
1243
1244 sub kvm_user_version {
1245     my ($binary) = @_;
1246
1247     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1248     my $st = stat($binary);
1249
1250     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1251     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1252         $cachedmtime == $st->mtime;
1253
1254     $kvm_user_version->{$binary} = 'unknown';
1255     $kvm_mtime->{$binary} = $st->mtime;
1256
1257     my $code = sub {
1258         my $line = shift;
1259         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1260             $kvm_user_version->{$binary} = $2;
1261         }
1262     };
1263
1264     eval { run_command([$binary, '--version'], outfunc => $code); };
1265     warn $@ if $@;
1266
1267     return $kvm_user_version->{$binary};
1268
1269 }
1270 my sub extract_version {
1271     my ($machine_type, $version) = @_;
1272     $version = kvm_user_version() if !defined($version);
1273     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1274 }
1275
1276 sub kernel_has_vhost_net {
1277     return -c '/dev/vhost-net';
1278 }
1279
1280 sub option_exists {
1281     my $key = shift;
1282     return defined($confdesc->{$key});
1283 }
1284
1285 my $cdrom_path;
1286 sub get_cdrom_path {
1287
1288     return $cdrom_path if defined($cdrom_path);
1289
1290     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1291
1292     if (!defined($cdrom_path)) {
1293         log_warn("no physical CD-ROM available, ignoring");
1294         $cdrom_path = '';
1295     }
1296
1297     return $cdrom_path;
1298 }
1299
1300 sub get_iso_path {
1301     my ($storecfg, $vmid, $cdrom) = @_;
1302
1303     if ($cdrom eq 'cdrom') {
1304         return get_cdrom_path();
1305     } elsif ($cdrom eq 'none') {
1306         return '';
1307     } elsif ($cdrom =~ m|^/|) {
1308         return $cdrom;
1309     } else {
1310         return PVE::Storage::path($storecfg, $cdrom);
1311     }
1312 }
1313
1314 # try to convert old style file names to volume IDs
1315 sub filename_to_volume_id {
1316     my ($vmid, $file, $media) = @_;
1317
1318      if (!($file eq 'none' || $file eq 'cdrom' ||
1319           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1320
1321         return if $file =~ m|/|;
1322
1323         if ($media && $media eq 'cdrom') {
1324             $file = "local:iso/$file";
1325         } else {
1326             $file = "local:$vmid/$file";
1327         }
1328     }
1329
1330     return $file;
1331 }
1332
1333 sub verify_media_type {
1334     my ($opt, $vtype, $media) = @_;
1335
1336     return if !$media;
1337
1338     my $etype;
1339     if ($media eq 'disk') {
1340         $etype = 'images';
1341     } elsif ($media eq 'cdrom') {
1342         $etype = 'iso';
1343     } else {
1344         die "internal error";
1345     }
1346
1347     return if ($vtype eq $etype);
1348
1349     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1350 }
1351
1352 sub cleanup_drive_path {
1353     my ($opt, $storecfg, $drive) = @_;
1354
1355     # try to convert filesystem paths to volume IDs
1356
1357     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1358         ($drive->{file} !~ m|^/dev/.+|) &&
1359         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1360         ($drive->{file} !~ m/^\d+$/)) {
1361         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1362         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1363             if !$vtype;
1364         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1365         verify_media_type($opt, $vtype, $drive->{media});
1366         $drive->{file} = $volid;
1367     }
1368
1369     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1370 }
1371
1372 sub parse_hotplug_features {
1373     my ($data) = @_;
1374
1375     my $res = {};
1376
1377     return $res if $data eq '0';
1378
1379     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1380
1381     foreach my $feature (PVE::Tools::split_list($data)) {
1382         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1383             $res->{$1} = 1;
1384         } else {
1385             die "invalid hotplug feature '$feature'\n";
1386         }
1387     }
1388     return $res;
1389 }
1390
1391 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1392 sub pve_verify_hotplug_features {
1393     my ($value, $noerr) = @_;
1394
1395     return $value if parse_hotplug_features($value);
1396
1397     return if $noerr;
1398
1399     die "unable to parse hotplug option\n";
1400 }
1401
1402 sub scsi_inquiry {
1403     my($fh, $noerr) = @_;
1404
1405     my $SG_IO = 0x2285;
1406     my $SG_GET_VERSION_NUM = 0x2282;
1407
1408     my $versionbuf = "\x00" x 8;
1409     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1410     if (!$ret) {
1411         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1412         return;
1413     }
1414     my $version = unpack("I", $versionbuf);
1415     if ($version < 30000) {
1416         die "scsi generic interface too old\n"  if !$noerr;
1417         return;
1418     }
1419
1420     my $buf = "\x00" x 36;
1421     my $sensebuf = "\x00" x 8;
1422     my $cmd = pack("C x3 C x1", 0x12, 36);
1423
1424     # see /usr/include/scsi/sg.h
1425     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1426
1427     my $packet = pack(
1428         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1429     );
1430
1431     $ret = ioctl($fh, $SG_IO, $packet);
1432     if (!$ret) {
1433         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1434         return;
1435     }
1436
1437     my @res = unpack($sg_io_hdr_t, $packet);
1438     if ($res[17] || $res[18]) {
1439         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1440         return;
1441     }
1442
1443     my $res = {};
1444     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1445
1446     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1447     $res->{type} &= 0x1F;
1448
1449     return $res;
1450 }
1451
1452 sub path_is_scsi {
1453     my ($path) = @_;
1454
1455     my $fh = IO::File->new("+<$path") || return;
1456     my $res = scsi_inquiry($fh, 1);
1457     close($fh);
1458
1459     return $res;
1460 }
1461
1462 sub print_tabletdevice_full {
1463     my ($conf, $arch) = @_;
1464
1465     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1466
1467     # we use uhci for old VMs because tablet driver was buggy in older qemu
1468     my $usbbus;
1469     if ($q35 || $arch eq 'aarch64') {
1470         $usbbus = 'ehci';
1471     } else {
1472         $usbbus = 'uhci';
1473     }
1474
1475     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1476 }
1477
1478 sub print_keyboarddevice_full {
1479     my ($conf, $arch) = @_;
1480
1481     return if $arch ne 'aarch64';
1482
1483     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1484 }
1485
1486 my sub get_drive_id {
1487     my ($drive) = @_;
1488     return "$drive->{interface}$drive->{index}";
1489 }
1490
1491 sub print_drivedevice_full {
1492     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1493
1494     my $device = '';
1495     my $maxdev = 0;
1496
1497     my $drive_id = get_drive_id($drive);
1498     if ($drive->{interface} eq 'virtio') {
1499         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1500         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1501         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1502     } elsif ($drive->{interface} eq 'scsi') {
1503
1504         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1505         my $unit = $drive->{index} % $maxdev;
1506         my $devicetype = 'hd';
1507         my $path = '';
1508         if (drive_is_cdrom($drive)) {
1509             $devicetype = 'cd';
1510         } else {
1511             if ($drive->{file} =~ m|^/|) {
1512                 $path = $drive->{file};
1513                 if (my $info = path_is_scsi($path)) {
1514                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1515                         $devicetype = 'block';
1516                     } elsif ($info->{type} == 1) { # tape
1517                         $devicetype = 'generic';
1518                     }
1519                 }
1520             } else {
1521                  $path = PVE::Storage::path($storecfg, $drive->{file});
1522             }
1523
1524             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1525             my $version = extract_version($machine_type, kvm_user_version());
1526             if ($path =~ m/^iscsi\:\/\// &&
1527                !min_version($version, 4, 1)) {
1528                 $devicetype = 'generic';
1529             }
1530         }
1531
1532         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1533             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1534         } else {
1535             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1536                 .",lun=$drive->{index}";
1537         }
1538         $device .= ",drive=drive-$drive_id,id=$drive_id";
1539
1540         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1541             $device .= ",rotation_rate=1";
1542         }
1543         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1544
1545     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1546         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1547         my $controller = int($drive->{index} / $maxdev);
1548         my $unit = $drive->{index} % $maxdev;
1549         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1550
1551         $device = "ide-$devicetype";
1552         if ($drive->{interface} eq 'ide') {
1553             $device .= ",bus=ide.$controller,unit=$unit";
1554         } else {
1555             $device .= ",bus=ahci$controller.$unit";
1556         }
1557         $device .= ",drive=drive-$drive_id,id=$drive_id";
1558
1559         if ($devicetype eq 'hd') {
1560             if (my $model = $drive->{model}) {
1561                 $model = URI::Escape::uri_unescape($model);
1562                 $device .= ",model=$model";
1563             }
1564             if ($drive->{ssd}) {
1565                 $device .= ",rotation_rate=1";
1566             }
1567         }
1568         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1569     } elsif ($drive->{interface} eq 'usb') {
1570         die "implement me";
1571         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1572     } else {
1573         die "unsupported interface type";
1574     }
1575
1576     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1577
1578     if (my $serial = $drive->{serial}) {
1579         $serial = URI::Escape::uri_unescape($serial);
1580         $device .= ",serial=$serial";
1581     }
1582
1583
1584     return $device;
1585 }
1586
1587 sub get_initiator_name {
1588     my $initiator;
1589
1590     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1591     while (defined(my $line = <$fh>)) {
1592         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1593         $initiator = $1;
1594         last;
1595     }
1596     $fh->close();
1597
1598     return $initiator;
1599 }
1600
1601 my sub storage_allows_io_uring_default {
1602     my ($scfg, $cache_direct) = @_;
1603
1604     # io_uring with cache mode writeback or writethrough on krbd will hang...
1605     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1606
1607     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1608     # sometimes, just plain disable...
1609     return if $scfg && $scfg->{type} eq 'lvm';
1610
1611     # io_uring causes problems when used with CIFS since kernel 5.15
1612     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1613     return if $scfg && $scfg->{type} eq 'cifs';
1614
1615     return 1;
1616 }
1617
1618 sub print_drive_commandline_full {
1619     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1620
1621     my $path;
1622     my $volid = $drive->{file};
1623     my $format = $drive->{format};
1624     my $drive_id = get_drive_id($drive);
1625
1626     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1627     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1628
1629     if (drive_is_cdrom($drive)) {
1630         $path = get_iso_path($storecfg, $vmid, $volid);
1631         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1632     } else {
1633         if ($storeid) {
1634             $path = PVE::Storage::path($storecfg, $volid);
1635             $format //= qemu_img_format($scfg, $volname);
1636         } else {
1637             $path = $volid;
1638             $format //= "raw";
1639         }
1640    }
1641
1642    my $is_rbd = $path =~ m/^rbd:/;
1643
1644     my $opts = '';
1645     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1646     foreach my $o (@qemu_drive_options) {
1647         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1648     }
1649
1650     # snapshot only accepts on|off
1651     if (defined($drive->{snapshot})) {
1652         my $v = $drive->{snapshot} ? 'on' : 'off';
1653         $opts .= ",snapshot=$v";
1654     }
1655
1656     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1657         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1658     }
1659
1660     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1661         my ($dir, $qmpname) = @$type;
1662         if (my $v = $drive->{"mbps$dir"}) {
1663             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1664         }
1665         if (my $v = $drive->{"mbps${dir}_max"}) {
1666             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1667         }
1668         if (my $v = $drive->{"bps${dir}_max_length"}) {
1669             $opts .= ",throttling.bps$qmpname-max-length=$v";
1670         }
1671         if (my $v = $drive->{"iops${dir}"}) {
1672             $opts .= ",throttling.iops$qmpname=$v";
1673         }
1674         if (my $v = $drive->{"iops${dir}_max"}) {
1675             $opts .= ",throttling.iops$qmpname-max=$v";
1676         }
1677         if (my $v = $drive->{"iops${dir}_max_length"}) {
1678             $opts .= ",throttling.iops$qmpname-max-length=$v";
1679         }
1680     }
1681
1682     if ($pbs_name) {
1683         $format = "rbd" if $is_rbd;
1684         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1685             if !$format;
1686         $opts .= ",format=alloc-track,file.driver=$format";
1687     } elsif ($format) {
1688         $opts .= ",format=$format";
1689     }
1690
1691     my $cache_direct = 0;
1692
1693     if (my $cache = $drive->{cache}) {
1694         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1695     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1696         $opts .= ",cache=none";
1697         $cache_direct = 1;
1698     }
1699
1700     if (!$drive->{aio}) {
1701         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1702             # io_uring supports all cache modes
1703             $opts .= ",aio=io_uring";
1704         } else {
1705             # aio native works only with O_DIRECT
1706             if($cache_direct) {
1707                 $opts .= ",aio=native";
1708             } else {
1709                 $opts .= ",aio=threads";
1710             }
1711         }
1712     }
1713
1714     if (!drive_is_cdrom($drive)) {
1715         my $detectzeroes;
1716         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1717             $detectzeroes = 'off';
1718         } elsif ($drive->{discard}) {
1719             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1720         } else {
1721             # This used to be our default with discard not being specified:
1722             $detectzeroes = 'on';
1723         }
1724
1725         # note: 'detect-zeroes' works per blockdev and we want it to persist
1726         # after the alloc-track is removed, so put it on 'file' directly
1727         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1728         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1729     }
1730
1731     if ($pbs_name) {
1732         $opts .= ",backing=$pbs_name";
1733         $opts .= ",auto-remove=on";
1734     }
1735
1736     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1737     my $file_param = "file";
1738     if ($pbs_name) {
1739         # non-rbd drivers require the underlying file to be a seperate block
1740         # node, so add a second .file indirection
1741         $file_param .= ".file" if !$is_rbd;
1742         $file_param .= ".filename";
1743     }
1744     my $pathinfo = $path ? "$file_param=$path," : '';
1745
1746     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1747 }
1748
1749 sub print_pbs_blockdev {
1750     my ($pbs_conf, $pbs_name) = @_;
1751     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1752     $blockdev .= ",repository=$pbs_conf->{repository}";
1753     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1754     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1755     $blockdev .= ",archive=$pbs_conf->{archive}";
1756     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1757     return $blockdev;
1758 }
1759
1760 sub print_netdevice_full {
1761     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1762
1763     my $device = $net->{model};
1764     if ($net->{model} eq 'virtio') {
1765          $device = 'virtio-net-pci';
1766      };
1767
1768     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1769     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1770     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1771         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1772         # and out of each queue plus one config interrupt and control vector queue
1773         my $vectors = $net->{queues} * 2 + 2;
1774         $tmpstr .= ",vectors=$vectors,mq=on";
1775         if (min_version($machine_version, 7, 1)) {
1776             $tmpstr .= ",packed=on";
1777         }
1778     }
1779
1780     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1781         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1782     }
1783
1784     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1785
1786     if (my $mtu = $net->{mtu}) {
1787         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1788             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1789             if ($mtu == 1) {
1790                  $mtu = $bridge_mtu;
1791             } elsif ($mtu < 576) {
1792                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1793             } elsif ($mtu > $bridge_mtu) {
1794                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1795             }
1796             $tmpstr .= ",host_mtu=$mtu";
1797         } else {
1798             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1799         }
1800     }
1801
1802     if ($use_old_bios_files) {
1803         my $romfile;
1804         if ($device eq 'virtio-net-pci') {
1805             $romfile = 'pxe-virtio.rom';
1806         } elsif ($device eq 'e1000') {
1807             $romfile = 'pxe-e1000.rom';
1808         } elsif ($device eq 'e1000e') {
1809             $romfile = 'pxe-e1000e.rom';
1810         } elsif ($device eq 'ne2k') {
1811             $romfile = 'pxe-ne2k_pci.rom';
1812         } elsif ($device eq 'pcnet') {
1813             $romfile = 'pxe-pcnet.rom';
1814         } elsif ($device eq 'rtl8139') {
1815             $romfile = 'pxe-rtl8139.rom';
1816         }
1817         $tmpstr .= ",romfile=$romfile" if $romfile;
1818     }
1819
1820     return $tmpstr;
1821 }
1822
1823 sub print_netdev_full {
1824     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1825
1826     my $i = '';
1827     if ($netid =~ m/^net(\d+)$/) {
1828         $i = int($1);
1829     }
1830
1831     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1832
1833     my $ifname = "tap${vmid}i$i";
1834
1835     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1836     die "interface name '$ifname' is too long (max 15 character)\n"
1837         if length($ifname) >= 16;
1838
1839     my $vhostparam = '';
1840     if (is_native($arch)) {
1841         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1842     }
1843
1844     my $vmname = $conf->{name} || "vm$vmid";
1845
1846     my $netdev = "";
1847     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1848
1849     if ($net->{bridge}) {
1850         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1851             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1852     } else {
1853         $netdev = "type=user,id=$netid,hostname=$vmname";
1854     }
1855
1856     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1857
1858     return $netdev;
1859 }
1860
1861 my $vga_map = {
1862     'cirrus' => 'cirrus-vga',
1863     'std' => 'VGA',
1864     'vmware' => 'vmware-svga',
1865     'virtio' => 'virtio-vga',
1866     'virtio-gl' => 'virtio-vga-gl',
1867 };
1868
1869 sub print_vga_device {
1870     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1871
1872     my $type = $vga_map->{$vga->{type}};
1873     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1874         $type = 'virtio-gpu';
1875     }
1876     my $vgamem_mb = $vga->{memory};
1877
1878     my $max_outputs = '';
1879     if ($qxlnum) {
1880         $type = $id ? 'qxl' : 'qxl-vga';
1881
1882         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1883             # set max outputs so linux can have up to 4 qxl displays with one device
1884             if (min_version($machine_version, 4, 1)) {
1885                 $max_outputs = ",max_outputs=4";
1886             }
1887         }
1888     }
1889
1890     die "no devicetype for $vga->{type}\n" if !$type;
1891
1892     my $memory = "";
1893     if ($vgamem_mb) {
1894         if ($vga->{type} =~ /^virtio/) {
1895             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1896             $memory = ",max_hostmem=$bytes";
1897         } elsif ($qxlnum) {
1898             # from https://www.spice-space.org/multiple-monitors.html
1899             $memory = ",vgamem_mb=$vga->{memory}";
1900             my $ram = $vgamem_mb * 4;
1901             my $vram = $vgamem_mb * 2;
1902             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1903         } else {
1904             $memory = ",vgamem_mb=$vga->{memory}";
1905         }
1906     } elsif ($qxlnum && $id) {
1907         $memory = ",ram_size=67108864,vram_size=33554432";
1908     }
1909
1910     my $edidoff = "";
1911     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1912         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1913     }
1914
1915     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1916     my $vgaid = "vga" . ($id // '');
1917     my $pciaddr;
1918     if ($q35 && $vgaid eq 'vga') {
1919         # the first display uses pcie.0 bus on q35 machines
1920         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1921     } else {
1922         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1923     }
1924
1925     if ($vga->{type} eq 'virtio-gl') {
1926         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1927         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1928             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1929
1930         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1931             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1932     }
1933
1934     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1935 }
1936
1937 sub parse_number_sets {
1938     my ($set) = @_;
1939     my $res = [];
1940     foreach my $part (split(/;/, $set)) {
1941         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1942             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1943             push @$res, [ $1, $2 ];
1944         } else {
1945             die "invalid range: $part\n";
1946         }
1947     }
1948     return $res;
1949 }
1950
1951 sub parse_numa {
1952     my ($data) = @_;
1953
1954     my $res = parse_property_string($numa_fmt, $data);
1955     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1956     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1957     return $res;
1958 }
1959
1960 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1961 sub parse_net {
1962     my ($data, $disable_mac_autogen) = @_;
1963
1964     my $res = eval { parse_property_string($net_fmt, $data) };
1965     if ($@) {
1966         warn $@;
1967         return;
1968     }
1969     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1970         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1971         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1972     }
1973     return $res;
1974 }
1975
1976 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1977 sub parse_ipconfig {
1978     my ($data) = @_;
1979
1980     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1981     if ($@) {
1982         warn $@;
1983         return;
1984     }
1985
1986     if ($res->{gw} && !$res->{ip}) {
1987         warn 'gateway specified without specifying an IP address';
1988         return;
1989     }
1990     if ($res->{gw6} && !$res->{ip6}) {
1991         warn 'IPv6 gateway specified without specifying an IPv6 address';
1992         return;
1993     }
1994     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1995         warn 'gateway specified together with DHCP';
1996         return;
1997     }
1998     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1999         # gw6 + auto/dhcp
2000         warn "IPv6 gateway specified together with $res->{ip6} address";
2001         return;
2002     }
2003
2004     if (!$res->{ip} && !$res->{ip6}) {
2005         return { ip => 'dhcp', ip6 => 'dhcp' };
2006     }
2007
2008     return $res;
2009 }
2010
2011 sub print_net {
2012     my $net = shift;
2013
2014     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2015 }
2016
2017 sub add_random_macs {
2018     my ($settings) = @_;
2019
2020     foreach my $opt (keys %$settings) {
2021         next if $opt !~ m/^net(\d+)$/;
2022         my $net = parse_net($settings->{$opt});
2023         next if !$net;
2024         $settings->{$opt} = print_net($net);
2025     }
2026 }
2027
2028 sub vm_is_volid_owner {
2029     my ($storecfg, $vmid, $volid) = @_;
2030
2031     if ($volid !~  m|^/|) {
2032         my ($path, $owner);
2033         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2034         if ($owner && ($owner == $vmid)) {
2035             return 1;
2036         }
2037     }
2038
2039     return;
2040 }
2041
2042 sub vmconfig_register_unused_drive {
2043     my ($storecfg, $vmid, $conf, $drive) = @_;
2044
2045     if (drive_is_cloudinit($drive)) {
2046         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2047         warn $@ if $@;
2048         delete $conf->{cloudinit};
2049     } elsif (!drive_is_cdrom($drive)) {
2050         my $volid = $drive->{file};
2051         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2052             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2053         }
2054     }
2055 }
2056
2057 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2058 my $smbios1_fmt = {
2059     uuid => {
2060         type => 'string',
2061         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2062         format_description => 'UUID',
2063         description => "Set SMBIOS1 UUID.",
2064         optional => 1,
2065     },
2066     version => {
2067         type => 'string',
2068         pattern => '[A-Za-z0-9+\/]+={0,2}',
2069         format_description => 'Base64 encoded string',
2070         description => "Set SMBIOS1 version.",
2071         optional => 1,
2072     },
2073     serial => {
2074         type => 'string',
2075         pattern => '[A-Za-z0-9+\/]+={0,2}',
2076         format_description => 'Base64 encoded string',
2077         description => "Set SMBIOS1 serial number.",
2078         optional => 1,
2079     },
2080     manufacturer => {
2081         type => 'string',
2082         pattern => '[A-Za-z0-9+\/]+={0,2}',
2083         format_description => 'Base64 encoded string',
2084         description => "Set SMBIOS1 manufacturer.",
2085         optional => 1,
2086     },
2087     product => {
2088         type => 'string',
2089         pattern => '[A-Za-z0-9+\/]+={0,2}',
2090         format_description => 'Base64 encoded string',
2091         description => "Set SMBIOS1 product ID.",
2092         optional => 1,
2093     },
2094     sku => {
2095         type => 'string',
2096         pattern => '[A-Za-z0-9+\/]+={0,2}',
2097         format_description => 'Base64 encoded string',
2098         description => "Set SMBIOS1 SKU string.",
2099         optional => 1,
2100     },
2101     family => {
2102         type => 'string',
2103         pattern => '[A-Za-z0-9+\/]+={0,2}',
2104         format_description => 'Base64 encoded string',
2105         description => "Set SMBIOS1 family string.",
2106         optional => 1,
2107     },
2108     base64 => {
2109         type => 'boolean',
2110         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2111         optional => 1,
2112     },
2113 };
2114
2115 sub parse_smbios1 {
2116     my ($data) = @_;
2117
2118     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2119     warn $@ if $@;
2120     return $res;
2121 }
2122
2123 sub print_smbios1 {
2124     my ($smbios1) = @_;
2125     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2126 }
2127
2128 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2129
2130 sub parse_watchdog {
2131     my ($value) = @_;
2132
2133     return if !$value;
2134
2135     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2136     warn $@ if $@;
2137     return $res;
2138 }
2139
2140 sub parse_guest_agent {
2141     my ($conf) = @_;
2142
2143     return {} if !defined($conf->{agent});
2144
2145     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2146     warn $@ if $@;
2147
2148     # if the agent is disabled ignore the other potentially set properties
2149     return {} if !$res->{enabled};
2150     return $res;
2151 }
2152
2153 sub get_qga_key {
2154     my ($conf, $key) = @_;
2155     return undef if !defined($conf->{agent});
2156
2157     my $agent = parse_guest_agent($conf);
2158     return $agent->{$key};
2159 }
2160
2161 sub parse_vga {
2162     my ($value) = @_;
2163
2164     return {} if !$value;
2165     my $res = eval { parse_property_string($vga_fmt, $value) };
2166     warn $@ if $@;
2167     return $res;
2168 }
2169
2170 sub parse_rng {
2171     my ($value) = @_;
2172
2173     return if !$value;
2174
2175     my $res = eval { parse_property_string($rng_fmt, $value) };
2176     warn $@ if $@;
2177     return $res;
2178 }
2179
2180 sub parse_meta_info {
2181     my ($value) = @_;
2182
2183     return if !$value;
2184
2185     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2186     warn $@ if $@;
2187     return $res;
2188 }
2189
2190 sub new_meta_info_string {
2191     my () = @_; # for now do not allow to override any value
2192
2193     return PVE::JSONSchema::print_property_string(
2194         {
2195             'creation-qemu' => kvm_user_version(),
2196             ctime => "". int(time()),
2197         },
2198         $meta_info_fmt
2199     );
2200 }
2201
2202 sub qemu_created_version_fixups {
2203     my ($conf, $forcemachine, $kvmver) = @_;
2204
2205     my $meta = parse_meta_info($conf->{meta}) // {};
2206     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2207
2208     # check if we need to apply some handling for VMs that always use the latest machine version but
2209     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2210     # would be required (we do not want to pin machine version for non-windows OS type)
2211     if (
2212         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2213         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2214         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2215         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2216     ) {
2217         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2218         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2219             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2220             # and thus with the predictable interface naming of systemd
2221             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2222         }
2223     }
2224     return;
2225 }
2226
2227 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2228 sub verify_usb_device {
2229     my ($value, $noerr) = @_;
2230
2231     return $value if parse_usb_device($value);
2232
2233     return if $noerr;
2234
2235     die "unable to parse usb device\n";
2236 }
2237
2238 # add JSON properties for create and set function
2239 sub json_config_properties {
2240     my ($prop, $with_disk_alloc) = @_;
2241
2242     my $skip_json_config_opts = {
2243         parent => 1,
2244         snaptime => 1,
2245         vmstate => 1,
2246         runningmachine => 1,
2247         runningcpu => 1,
2248         meta => 1,
2249     };
2250
2251     foreach my $opt (keys %$confdesc) {
2252         next if $skip_json_config_opts->{$opt};
2253
2254         if ($with_disk_alloc && is_valid_drivename($opt)) {
2255             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2256         } else {
2257             $prop->{$opt} = $confdesc->{$opt};
2258         }
2259     }
2260
2261     return $prop;
2262 }
2263
2264 # Properties that we can read from an OVF file
2265 sub json_ovf_properties {
2266     my $prop = {};
2267
2268     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2269         $prop->{$device} = {
2270             type => 'string',
2271             format => 'pve-volume-id-or-absolute-path',
2272             description => "Disk image that gets imported to $device",
2273             optional => 1,
2274         };
2275     }
2276
2277     $prop->{cores} = {
2278         type => 'integer',
2279         description => "The number of CPU cores.",
2280         optional => 1,
2281     };
2282     $prop->{memory} = {
2283         type => 'integer',
2284         description => "Amount of RAM for the VM in MB.",
2285         optional => 1,
2286     };
2287     $prop->{name} = {
2288         type => 'string',
2289         description => "Name of the VM.",
2290         optional => 1,
2291     };
2292
2293     return $prop;
2294 }
2295
2296 # return copy of $confdesc_cloudinit to generate documentation
2297 sub cloudinit_config_properties {
2298
2299     return dclone($confdesc_cloudinit);
2300 }
2301
2302 sub cloudinit_pending_properties {
2303     my $p = {
2304         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2305         name => 1,
2306     };
2307     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2308     return $p;
2309 }
2310
2311 sub check_type {
2312     my ($key, $value) = @_;
2313
2314     die "unknown setting '$key'\n" if !$confdesc->{$key};
2315
2316     my $type = $confdesc->{$key}->{type};
2317
2318     if (!defined($value)) {
2319         die "got undefined value\n";
2320     }
2321
2322     if ($value =~ m/[\n\r]/) {
2323         die "property contains a line feed\n";
2324     }
2325
2326     if ($type eq 'boolean') {
2327         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2328         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2329         die "type check ('boolean') failed - got '$value'\n";
2330     } elsif ($type eq 'integer') {
2331         return int($1) if $value =~ m/^(\d+)$/;
2332         die "type check ('integer') failed - got '$value'\n";
2333     } elsif ($type eq 'number') {
2334         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2335         die "type check ('number') failed - got '$value'\n";
2336     } elsif ($type eq 'string') {
2337         if (my $fmt = $confdesc->{$key}->{format}) {
2338             PVE::JSONSchema::check_format($fmt, $value);
2339             return $value;
2340         }
2341         $value =~ s/^\"(.*)\"$/$1/;
2342         return $value;
2343     } else {
2344         die "internal error"
2345     }
2346 }
2347
2348 sub destroy_vm {
2349     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2350
2351     my $conf = PVE::QemuConfig->load_config($vmid);
2352
2353     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2354         PVE::QemuConfig->check_lock($conf);
2355     }
2356
2357     if ($conf->{template}) {
2358         # check if any base image is still used by a linked clone
2359         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2360                 my ($ds, $drive) = @_;
2361                 return if drive_is_cdrom($drive);
2362
2363                 my $volid = $drive->{file};
2364                 return if !$volid || $volid =~ m|^/|;
2365
2366                 die "base volume '$volid' is still in use by linked cloned\n"
2367                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2368
2369         });
2370     }
2371
2372     my $volids = {};
2373     my $remove_owned_drive = sub {
2374         my ($ds, $drive) = @_;
2375         return if drive_is_cdrom($drive, 1);
2376
2377         my $volid = $drive->{file};
2378         return if !$volid || $volid =~ m|^/|;
2379         return if $volids->{$volid};
2380
2381         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2382         return if !$path || !$owner || ($owner != $vmid);
2383
2384         $volids->{$volid} = 1;
2385         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2386         warn "Could not remove disk '$volid', check manually: $@" if $@;
2387     };
2388
2389     # only remove disks owned by this VM (referenced in the config)
2390     my $include_opts = {
2391         include_unused => 1,
2392         extra_keys => ['vmstate'],
2393     };
2394     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2395
2396     for my $snap (values %{$conf->{snapshots}}) {
2397         next if !defined($snap->{vmstate});
2398         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2399         next if !defined($drive);
2400         $remove_owned_drive->('vmstate', $drive);
2401     }
2402
2403     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2404
2405     if ($purge_unreferenced) { # also remove unreferenced disk
2406         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2407         PVE::Storage::foreach_volid($vmdisks, sub {
2408             my ($volid, $sid, $volname, $d) = @_;
2409             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2410             warn $@ if $@;
2411         });
2412     }
2413
2414     if (defined $replacement_conf) {
2415         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2416     } else {
2417         PVE::QemuConfig->destroy_config($vmid);
2418     }
2419 }
2420
2421 sub parse_vm_config {
2422     my ($filename, $raw, $strict) = @_;
2423
2424     return if !defined($raw);
2425
2426     my $res = {
2427         digest => Digest::SHA::sha1_hex($raw),
2428         snapshots => {},
2429         pending => {},
2430         cloudinit => {},
2431     };
2432
2433     my $handle_error = sub {
2434         my ($msg) = @_;
2435
2436         if ($strict) {
2437             die $msg;
2438         } else {
2439             warn $msg;
2440         }
2441     };
2442
2443     $filename =~ m|/qemu-server/(\d+)\.conf$|
2444         || die "got strange filename '$filename'";
2445
2446     my $vmid = $1;
2447
2448     my $conf = $res;
2449     my $descr;
2450     my $finish_description = sub {
2451         if (defined($descr)) {
2452             $descr =~ s/\s+$//;
2453             $conf->{description} = $descr;
2454         }
2455         $descr = undef;
2456     };
2457     my $section = '';
2458
2459     my @lines = split(/\n/, $raw);
2460     foreach my $line (@lines) {
2461         next if $line =~ m/^\s*$/;
2462
2463         if ($line =~ m/^\[PENDING\]\s*$/i) {
2464             $section = 'pending';
2465             $finish_description->();
2466             $conf = $res->{$section} = {};
2467             next;
2468         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2469             $section = 'cloudinit';
2470             $finish_description->();
2471             $conf = $res->{$section} = {};
2472             next;
2473
2474         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2475             $section = $1;
2476             $finish_description->();
2477             $conf = $res->{snapshots}->{$section} = {};
2478             next;
2479         }
2480
2481         if ($line =~ m/^\#(.*)$/) {
2482             $descr = '' if !defined($descr);
2483             $descr .= PVE::Tools::decode_text($1) . "\n";
2484             next;
2485         }
2486
2487         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2488             $descr = '' if !defined($descr);
2489             $descr .= PVE::Tools::decode_text($2);
2490         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2491             $conf->{snapstate} = $1;
2492         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2493             my $key = $1;
2494             my $value = $2;
2495             $conf->{$key} = $value;
2496         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2497             my $value = $1;
2498             if ($section eq 'pending') {
2499                 $conf->{delete} = $value; # we parse this later
2500             } else {
2501                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2502             }
2503         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2504             my $key = $1;
2505             my $value = $2;
2506             if ($section eq 'cloudinit') {
2507                 # ignore validation only used for informative purpose
2508                 $conf->{$key} = $value;
2509                 next;
2510             }
2511             eval { $value = check_type($key, $value); };
2512             if ($@) {
2513                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2514             } else {
2515                 $key = 'ide2' if $key eq 'cdrom';
2516                 my $fmt = $confdesc->{$key}->{format};
2517                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2518                     my $v = parse_drive($key, $value);
2519                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2520                         $v->{file} = $volid;
2521                         $value = print_drive($v);
2522                     } else {
2523                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2524                         next;
2525                     }
2526                 }
2527
2528                 $conf->{$key} = $value;
2529             }
2530         } else {
2531             $handle_error->("vm $vmid - unable to parse config: $line\n");
2532         }
2533     }
2534
2535     $finish_description->();
2536     delete $res->{snapstate}; # just to be sure
2537
2538     return $res;
2539 }
2540
2541 sub write_vm_config {
2542     my ($filename, $conf) = @_;
2543
2544     delete $conf->{snapstate}; # just to be sure
2545
2546     if ($conf->{cdrom}) {
2547         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2548         $conf->{ide2} = $conf->{cdrom};
2549         delete $conf->{cdrom};
2550     }
2551
2552     # we do not use 'smp' any longer
2553     if ($conf->{sockets}) {
2554         delete $conf->{smp};
2555     } elsif ($conf->{smp}) {
2556         $conf->{sockets} = $conf->{smp};
2557         delete $conf->{cores};
2558         delete $conf->{smp};
2559     }
2560
2561     my $used_volids = {};
2562
2563     my $cleanup_config = sub {
2564         my ($cref, $pending, $snapname) = @_;
2565
2566         foreach my $key (keys %$cref) {
2567             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2568                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2569             my $value = $cref->{$key};
2570             if ($key eq 'delete') {
2571                 die "propertry 'delete' is only allowed in [PENDING]\n"
2572                     if !$pending;
2573                 # fixme: check syntax?
2574                 next;
2575             }
2576             eval { $value = check_type($key, $value); };
2577             die "unable to parse value of '$key' - $@" if $@;
2578
2579             $cref->{$key} = $value;
2580
2581             if (!$snapname && is_valid_drivename($key)) {
2582                 my $drive = parse_drive($key, $value);
2583                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2584             }
2585         }
2586     };
2587
2588     &$cleanup_config($conf);
2589
2590     &$cleanup_config($conf->{pending}, 1);
2591
2592     foreach my $snapname (keys %{$conf->{snapshots}}) {
2593         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2594         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2595     }
2596
2597     # remove 'unusedX' settings if we re-add a volume
2598     foreach my $key (keys %$conf) {
2599         my $value = $conf->{$key};
2600         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2601             delete $conf->{$key};
2602         }
2603     }
2604
2605     my $generate_raw_config = sub {
2606         my ($conf, $pending) = @_;
2607
2608         my $raw = '';
2609
2610         # add description as comment to top of file
2611         if (defined(my $descr = $conf->{description})) {
2612             if ($descr) {
2613                 foreach my $cl (split(/\n/, $descr)) {
2614                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2615                 }
2616             } else {
2617                 $raw .= "#\n" if $pending;
2618             }
2619         }
2620
2621         foreach my $key (sort keys %$conf) {
2622             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2623             $raw .= "$key: $conf->{$key}\n";
2624         }
2625         return $raw;
2626     };
2627
2628     my $raw = &$generate_raw_config($conf);
2629
2630     if (scalar(keys %{$conf->{pending}})){
2631         $raw .= "\n[PENDING]\n";
2632         $raw .= &$generate_raw_config($conf->{pending}, 1);
2633     }
2634
2635     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2636         $raw .= "\n[special:cloudinit]\n";
2637         $raw .= &$generate_raw_config($conf->{cloudinit});
2638     }
2639
2640     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2641         $raw .= "\n[$snapname]\n";
2642         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2643     }
2644
2645     return $raw;
2646 }
2647
2648 sub load_defaults {
2649
2650     my $res = {};
2651
2652     # we use static defaults from our JSON schema configuration
2653     foreach my $key (keys %$confdesc) {
2654         if (defined(my $default = $confdesc->{$key}->{default})) {
2655             $res->{$key} = $default;
2656         }
2657     }
2658
2659     return $res;
2660 }
2661
2662 sub config_list {
2663     my $vmlist = PVE::Cluster::get_vmlist();
2664     my $res = {};
2665     return $res if !$vmlist || !$vmlist->{ids};
2666     my $ids = $vmlist->{ids};
2667     my $nodename = nodename();
2668
2669     foreach my $vmid (keys %$ids) {
2670         my $d = $ids->{$vmid};
2671         next if !$d->{node} || $d->{node} ne $nodename;
2672         next if !$d->{type} || $d->{type} ne 'qemu';
2673         $res->{$vmid}->{exists} = 1;
2674     }
2675     return $res;
2676 }
2677
2678 # test if VM uses local resources (to prevent migration)
2679 sub check_local_resources {
2680     my ($conf, $noerr) = @_;
2681
2682     my @loc_res = ();
2683
2684     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2685     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2686
2687     push @loc_res, "ivshmem" if $conf->{ivshmem};
2688
2689     foreach my $k (keys %$conf) {
2690         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2691         # sockets are safe: they will recreated be on the target side post-migrate
2692         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2693         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2694     }
2695
2696     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2697
2698     return \@loc_res;
2699 }
2700
2701 # check if used storages are available on all nodes (use by migrate)
2702 sub check_storage_availability {
2703     my ($storecfg, $conf, $node) = @_;
2704
2705     PVE::QemuConfig->foreach_volume($conf, sub {
2706         my ($ds, $drive) = @_;
2707
2708         my $volid = $drive->{file};
2709         return if !$volid;
2710
2711         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2712         return if !$sid;
2713
2714         # check if storage is available on both nodes
2715         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2716         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2717
2718         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2719
2720         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2721             if !$scfg->{content}->{$vtype};
2722    });
2723 }
2724
2725 # list nodes where all VM images are available (used by has_feature API)
2726 sub shared_nodes {
2727     my ($conf, $storecfg) = @_;
2728
2729     my $nodelist = PVE::Cluster::get_nodelist();
2730     my $nodehash = { map { $_ => 1 } @$nodelist };
2731     my $nodename = nodename();
2732
2733     PVE::QemuConfig->foreach_volume($conf, sub {
2734         my ($ds, $drive) = @_;
2735
2736         my $volid = $drive->{file};
2737         return if !$volid;
2738
2739         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2740         if ($storeid) {
2741             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2742             if ($scfg->{disable}) {
2743                 $nodehash = {};
2744             } elsif (my $avail = $scfg->{nodes}) {
2745                 foreach my $node (keys %$nodehash) {
2746                     delete $nodehash->{$node} if !$avail->{$node};
2747                 }
2748             } elsif (!$scfg->{shared}) {
2749                 foreach my $node (keys %$nodehash) {
2750                     delete $nodehash->{$node} if $node ne $nodename
2751                 }
2752             }
2753         }
2754     });
2755
2756     return $nodehash
2757 }
2758
2759 sub check_local_storage_availability {
2760     my ($conf, $storecfg) = @_;
2761
2762     my $nodelist = PVE::Cluster::get_nodelist();
2763     my $nodehash = { map { $_ => {} } @$nodelist };
2764
2765     PVE::QemuConfig->foreach_volume($conf, sub {
2766         my ($ds, $drive) = @_;
2767
2768         my $volid = $drive->{file};
2769         return if !$volid;
2770
2771         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2772         if ($storeid) {
2773             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2774
2775             if ($scfg->{disable}) {
2776                 foreach my $node (keys %$nodehash) {
2777                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2778                 }
2779             } elsif (my $avail = $scfg->{nodes}) {
2780                 foreach my $node (keys %$nodehash) {
2781                     if (!$avail->{$node}) {
2782                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2783                     }
2784                 }
2785             }
2786         }
2787     });
2788
2789     foreach my $node (values %$nodehash) {
2790         if (my $unavail = $node->{unavailable_storages}) {
2791             $node->{unavailable_storages} = [ sort keys %$unavail ];
2792         }
2793     }
2794
2795     return $nodehash
2796 }
2797
2798 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2799 sub check_running {
2800     my ($vmid, $nocheck, $node) = @_;
2801
2802     # $nocheck is set when called during a migration, in which case the config
2803     # file might still or already reside on the *other* node
2804     # - because rename has already happened, and current node is source
2805     # - because rename hasn't happened yet, and current node is target
2806     # - because rename has happened, current node is target, but hasn't yet
2807     # processed it yet
2808     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2809     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2810 }
2811
2812 sub vzlist {
2813
2814     my $vzlist = config_list();
2815
2816     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2817
2818     while (defined(my $de = $fd->read)) {
2819         next if $de !~ m/^(\d+)\.pid$/;
2820         my $vmid = $1;
2821         next if !defined($vzlist->{$vmid});
2822         if (my $pid = check_running($vmid)) {
2823             $vzlist->{$vmid}->{pid} = $pid;
2824         }
2825     }
2826
2827     return $vzlist;
2828 }
2829
2830 our $vmstatus_return_properties = {
2831     vmid => get_standard_option('pve-vmid'),
2832     status => {
2833         description => "QEMU process status.",
2834         type => 'string',
2835         enum => ['stopped', 'running'],
2836     },
2837     maxmem => {
2838         description => "Maximum memory in bytes.",
2839         type => 'integer',
2840         optional => 1,
2841         renderer => 'bytes',
2842     },
2843     maxdisk => {
2844         description => "Root disk size in bytes.",
2845         type => 'integer',
2846         optional => 1,
2847         renderer => 'bytes',
2848     },
2849     name => {
2850         description => "VM name.",
2851         type => 'string',
2852         optional => 1,
2853     },
2854     qmpstatus => {
2855         description => "QEMU QMP agent status.",
2856         type => 'string',
2857         optional => 1,
2858     },
2859     pid => {
2860         description => "PID of running qemu process.",
2861         type => 'integer',
2862         optional => 1,
2863     },
2864     uptime => {
2865         description => "Uptime.",
2866         type => 'integer',
2867         optional => 1,
2868         renderer => 'duration',
2869     },
2870     cpus => {
2871         description => "Maximum usable CPUs.",
2872         type => 'number',
2873         optional => 1,
2874     },
2875     lock => {
2876         description => "The current config lock, if any.",
2877         type => 'string',
2878         optional => 1,
2879     },
2880     tags => {
2881         description  => "The current configured tags, if any",
2882         type => 'string',
2883         optional => 1,
2884     },
2885     'running-machine' => {
2886         description => "The currently running machine type (if running).",
2887         type => 'string',
2888         optional => 1,
2889     },
2890     'running-qemu' => {
2891         description => "The currently running QEMU version (if running).",
2892         type => 'string',
2893         optional => 1,
2894     },
2895 };
2896
2897 my $last_proc_pid_stat;
2898
2899 # get VM status information
2900 # This must be fast and should not block ($full == false)
2901 # We only query KVM using QMP if $full == true (this can be slow)
2902 sub vmstatus {
2903     my ($opt_vmid, $full) = @_;
2904
2905     my $res = {};
2906
2907     my $storecfg = PVE::Storage::config();
2908
2909     my $list = vzlist();
2910     my $defaults = load_defaults();
2911
2912     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2913
2914     my $cpucount = $cpuinfo->{cpus} || 1;
2915
2916     foreach my $vmid (keys %$list) {
2917         next if $opt_vmid && ($vmid ne $opt_vmid);
2918
2919         my $conf = PVE::QemuConfig->load_config($vmid);
2920
2921         my $d = { vmid => int($vmid) };
2922         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2923
2924         # fixme: better status?
2925         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2926
2927         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2928         if (defined($size)) {
2929             $d->{disk} = 0; # no info available
2930             $d->{maxdisk} = $size;
2931         } else {
2932             $d->{disk} = 0;
2933             $d->{maxdisk} = 0;
2934         }
2935
2936         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2937             * ($conf->{cores} || $defaults->{cores});
2938         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2939         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2940
2941         $d->{name} = $conf->{name} || "VM $vmid";
2942         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2943             : $defaults->{memory}*(1024*1024);
2944
2945         if ($conf->{balloon}) {
2946             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2947             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2948                 : $defaults->{shares};
2949         }
2950
2951         $d->{uptime} = 0;
2952         $d->{cpu} = 0;
2953         $d->{mem} = 0;
2954
2955         $d->{netout} = 0;
2956         $d->{netin} = 0;
2957
2958         $d->{diskread} = 0;
2959         $d->{diskwrite} = 0;
2960
2961         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2962
2963         $d->{serial} = 1 if conf_has_serial($conf);
2964         $d->{lock} = $conf->{lock} if $conf->{lock};
2965         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2966
2967         $res->{$vmid} = $d;
2968     }
2969
2970     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2971     foreach my $dev (keys %$netdev) {
2972         next if $dev !~ m/^tap([1-9]\d*)i/;
2973         my $vmid = $1;
2974         my $d = $res->{$vmid};
2975         next if !$d;
2976
2977         $d->{netout} += $netdev->{$dev}->{receive};
2978         $d->{netin} += $netdev->{$dev}->{transmit};
2979
2980         if ($full) {
2981             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2982             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2983         }
2984
2985     }
2986
2987     my $ctime = gettimeofday;
2988
2989     foreach my $vmid (keys %$list) {
2990
2991         my $d = $res->{$vmid};
2992         my $pid = $d->{pid};
2993         next if !$pid;
2994
2995         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2996         next if !$pstat; # not running
2997
2998         my $used = $pstat->{utime} + $pstat->{stime};
2999
3000         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3001
3002         if ($pstat->{vsize}) {
3003             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3004         }
3005
3006         my $old = $last_proc_pid_stat->{$pid};
3007         if (!$old) {
3008             $last_proc_pid_stat->{$pid} = {
3009                 time => $ctime,
3010                 used => $used,
3011                 cpu => 0,
3012             };
3013             next;
3014         }
3015
3016         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3017
3018         if ($dtime > 1000) {
3019             my $dutime = $used -  $old->{used};
3020
3021             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3022             $last_proc_pid_stat->{$pid} = {
3023                 time => $ctime,
3024                 used => $used,
3025                 cpu => $d->{cpu},
3026             };
3027         } else {
3028             $d->{cpu} = $old->{cpu};
3029         }
3030     }
3031
3032     return $res if !$full;
3033
3034     my $qmpclient = PVE::QMPClient->new();
3035
3036     my $ballooncb = sub {
3037         my ($vmid, $resp) = @_;
3038
3039         my $info = $resp->{'return'};
3040         return if !$info->{max_mem};
3041
3042         my $d = $res->{$vmid};
3043
3044         # use memory assigned to VM
3045         $d->{maxmem} = $info->{max_mem};
3046         $d->{balloon} = $info->{actual};
3047
3048         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3049             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3050             $d->{freemem} = $info->{free_mem};
3051         }
3052
3053         $d->{ballooninfo} = $info;
3054     };
3055
3056     my $blockstatscb = sub {
3057         my ($vmid, $resp) = @_;
3058         my $data = $resp->{'return'} || [];
3059         my $totalrdbytes = 0;
3060         my $totalwrbytes = 0;
3061
3062         for my $blockstat (@$data) {
3063             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3064             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3065
3066             $blockstat->{device} =~ s/drive-//;
3067             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3068         }
3069         $res->{$vmid}->{diskread} = $totalrdbytes;
3070         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3071     };
3072
3073     my $machinecb = sub {
3074         my ($vmid, $resp) = @_;
3075         my $data = $resp->{'return'} || [];
3076
3077         $res->{$vmid}->{'running-machine'} =
3078             PVE::QemuServer::Machine::current_from_query_machines($data);
3079     };
3080
3081     my $versioncb = sub {
3082         my ($vmid, $resp) = @_;
3083         my $data = $resp->{'return'} // {};
3084         my $version = 'unknown';
3085
3086         if (my $v = $data->{qemu}) {
3087             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3088         }
3089
3090         $res->{$vmid}->{'running-qemu'} = $version;
3091     };
3092
3093     my $statuscb = sub {
3094         my ($vmid, $resp) = @_;
3095
3096         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3097         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3098         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3099         # this fails if ballon driver is not loaded, so this must be
3100         # the last commnand (following command are aborted if this fails).
3101         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3102
3103         my $status = 'unknown';
3104         if (!defined($status = $resp->{'return'}->{status})) {
3105             warn "unable to get VM status\n";
3106             return;
3107         }
3108
3109         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3110     };
3111
3112     foreach my $vmid (keys %$list) {
3113         next if $opt_vmid && ($vmid ne $opt_vmid);
3114         next if !$res->{$vmid}->{pid}; # not running
3115         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3116     }
3117
3118     $qmpclient->queue_execute(undef, 2);
3119
3120     foreach my $vmid (keys %$list) {
3121         next if $opt_vmid && ($vmid ne $opt_vmid);
3122         next if !$res->{$vmid}->{pid}; #not running
3123
3124         # we can't use the $qmpclient since it might have already aborted on
3125         # 'query-balloon', but this might also fail for older versions...
3126         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3127         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3128     }
3129
3130     foreach my $vmid (keys %$list) {
3131         next if $opt_vmid && ($vmid ne $opt_vmid);
3132         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3133     }
3134
3135     return $res;
3136 }
3137
3138 sub conf_has_serial {
3139     my ($conf) = @_;
3140
3141     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3142         if ($conf->{"serial$i"}) {
3143             return 1;
3144         }
3145     }
3146
3147     return 0;
3148 }
3149
3150 sub conf_has_audio {
3151     my ($conf, $id) = @_;
3152
3153     $id //= 0;
3154     my $audio = $conf->{"audio$id"};
3155     return if !defined($audio);
3156
3157     my $audioproperties = parse_property_string($audio_fmt, $audio);
3158     my $audiodriver = $audioproperties->{driver} // 'spice';
3159
3160     return {
3161         dev => $audioproperties->{device},
3162         dev_id => "audiodev$id",
3163         backend => $audiodriver,
3164         backend_id => "$audiodriver-backend${id}",
3165     };
3166 }
3167
3168 sub audio_devs {
3169     my ($audio, $audiopciaddr, $machine_version) = @_;
3170
3171     my $devs = [];
3172
3173     my $id = $audio->{dev_id};
3174     my $audiodev = "";
3175     if (min_version($machine_version, 4, 2)) {
3176         $audiodev = ",audiodev=$audio->{backend_id}";
3177     }
3178
3179     if ($audio->{dev} eq 'AC97') {
3180         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3181     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3182         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3183         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3184         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3185     } else {
3186         die "unkown audio device '$audio->{dev}', implement me!";
3187     }
3188
3189     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3190
3191     return $devs;
3192 }
3193
3194 sub get_tpm_paths {
3195     my ($vmid) = @_;
3196     return {
3197         socket => "/var/run/qemu-server/$vmid.swtpm",
3198         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3199     };
3200 }
3201
3202 sub add_tpm_device {
3203     my ($vmid, $devices, $conf) = @_;
3204
3205     return if !$conf->{tpmstate0};
3206
3207     my $paths = get_tpm_paths($vmid);
3208
3209     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3210     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3211     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3212 }
3213
3214 sub start_swtpm {
3215     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3216
3217     return if !$tpmdrive;
3218
3219     my $state;
3220     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3221     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3222     if ($storeid) {
3223         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3224     } else {
3225         $state = $tpm->{file};
3226     }
3227
3228     my $paths = get_tpm_paths($vmid);
3229
3230     # during migration, we will get state from remote
3231     #
3232     if (!$migration) {
3233         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3234         my $setup_cmd = [
3235             "swtpm_setup",
3236             "--tpmstate",
3237             "file://$state",
3238             "--createek",
3239             "--create-ek-cert",
3240             "--create-platform-cert",
3241             "--lock-nvram",
3242             "--config",
3243             "/etc/swtpm_setup.conf", # do not use XDG configs
3244             "--runas",
3245             "0", # force creation as root, error if not possible
3246             "--not-overwrite", # ignore existing state, do not modify
3247         ];
3248
3249         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3250         # TPM 2.0 supports ECC crypto, use if possible
3251         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3252
3253         run_command($setup_cmd, outfunc => sub {
3254             print "swtpm_setup: $1\n";
3255         });
3256     }
3257
3258     my $emulator_cmd = [
3259         "swtpm",
3260         "socket",
3261         "--tpmstate",
3262         "backend-uri=file://$state,mode=0600",
3263         "--ctrl",
3264         "type=unixio,path=$paths->{socket},mode=0600",
3265         "--pid",
3266         "file=$paths->{pid}",
3267         "--terminate", # terminate on QEMU disconnect
3268         "--daemon",
3269     ];
3270     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3271     run_command($emulator_cmd, outfunc => sub { print $1; });
3272
3273     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3274     while (! -e $paths->{pid}) {
3275         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3276         usleep(50_000);
3277     }
3278
3279     # return untainted PID of swtpm daemon so it can be killed on error
3280     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3281     return $1;
3282 }
3283
3284 sub vga_conf_has_spice {
3285     my ($vga) = @_;
3286
3287     my $vgaconf = parse_vga($vga);
3288     my $vgatype = $vgaconf->{type};
3289     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3290
3291     return $1 || 1;
3292 }
3293
3294 sub is_native($) {
3295     my ($arch) = @_;
3296     return get_host_arch() eq $arch;
3297 }
3298
3299 sub get_vm_arch {
3300     my ($conf) = @_;
3301     return $conf->{arch} // get_host_arch();
3302 }
3303
3304 my $default_machines = {
3305     x86_64 => 'pc',
3306     aarch64 => 'virt',
3307 };
3308
3309 sub get_installed_machine_version {
3310     my ($kvmversion) = @_;
3311     $kvmversion = kvm_user_version() if !defined($kvmversion);
3312     $kvmversion =~ m/^(\d+\.\d+)/;
3313     return $1;
3314 }
3315
3316 sub windows_get_pinned_machine_version {
3317     my ($machine, $base_version, $kvmversion) = @_;
3318
3319     my $pin_version = $base_version;
3320     if (!defined($base_version) ||
3321         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3322     ) {
3323         $pin_version = get_installed_machine_version($kvmversion);
3324     }
3325     if (!$machine || $machine eq 'pc') {
3326         $machine = "pc-i440fx-$pin_version";
3327     } elsif ($machine eq 'q35') {
3328         $machine = "pc-q35-$pin_version";
3329     } elsif ($machine eq 'virt') {
3330         $machine = "virt-$pin_version";
3331     } else {
3332         warn "unknown machine type '$machine', not touching that!\n";
3333     }
3334
3335     return $machine;
3336 }
3337
3338 sub get_vm_machine {
3339     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3340
3341     my $machine = $forcemachine || $conf->{machine};
3342
3343     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3344         $kvmversion //= kvm_user_version();
3345         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3346         # layout which confuses windows quite a bit and may result in various regressions..
3347         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3348         if (windows_version($conf->{ostype})) {
3349             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3350         }
3351         $arch //= 'x86_64';
3352         $machine ||= $default_machines->{$arch};
3353         if ($add_pve_version) {
3354             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3355             $machine .= "+pve$pvever";
3356         }
3357     }
3358
3359     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3360         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3361         $machine = $1 if $is_pxe;
3362
3363         # for version-pinned machines that do not include a pve-version (e.g.
3364         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3365         $machine .= '+pve0';
3366
3367         $machine .= '.pxe' if $is_pxe;
3368     }
3369
3370     return $machine;
3371 }
3372
3373 sub get_ovmf_files($$$) {
3374     my ($arch, $efidisk, $smm) = @_;
3375
3376     my $types = $OVMF->{$arch}
3377         or die "no OVMF images known for architecture '$arch'\n";
3378
3379     my $type = 'default';
3380     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3381         $type = $smm ? "4m" : "4m-no-smm";
3382         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3383     }
3384
3385     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3386     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3387     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3388
3389     return ($ovmf_code, $ovmf_vars);
3390 }
3391
3392 my $Arch2Qemu = {
3393     aarch64 => '/usr/bin/qemu-system-aarch64',
3394     x86_64 => '/usr/bin/qemu-system-x86_64',
3395 };
3396 sub get_command_for_arch($) {
3397     my ($arch) = @_;
3398     return '/usr/bin/kvm' if is_native($arch);
3399
3400     my $cmd = $Arch2Qemu->{$arch}
3401         or die "don't know how to emulate architecture '$arch'\n";
3402     return $cmd;
3403 }
3404
3405 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3406 # to use in a QEMU command line (-cpu element), first array_intersect the result
3407 # of query_supported_ with query_understood_. This is necessary because:
3408 #
3409 # a) query_understood_ returns flags the host cannot use and
3410 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3411 #    flags, but CPU settings - with most of them being flags. Those settings
3412 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3413 #
3414 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3415 # expensive. If you need the value returned from this, you can get it much
3416 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3417 # $accel being 'kvm' or 'tcg'.
3418 #
3419 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3420 # changes, automatically populating pmxcfs.
3421 #
3422 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3423 # since kvm and tcg machines support different flags
3424 #
3425 sub query_supported_cpu_flags {
3426     my ($arch) = @_;
3427
3428     $arch //= get_host_arch();
3429     my $default_machine = $default_machines->{$arch};
3430
3431     my $flags = {};
3432
3433     # FIXME: Once this is merged, the code below should work for ARM as well:
3434     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3435     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3436         $arch eq "aarch64";
3437
3438     my $kvm_supported = defined(kvm_version());
3439     my $qemu_cmd = get_command_for_arch($arch);
3440     my $fakevmid = -1;
3441     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3442
3443     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3444     my $query_supported_run_qemu = sub {
3445         my ($kvm) = @_;
3446
3447         my $flags = {};
3448         my $cmd = [
3449             $qemu_cmd,
3450             '-machine', $default_machine,
3451             '-display', 'none',
3452             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3453             '-mon', 'chardev=qmp,mode=control',
3454             '-pidfile', $pidfile,
3455             '-S', '-daemonize'
3456         ];
3457
3458         if (!$kvm) {
3459             push @$cmd, '-accel', 'tcg';
3460         }
3461
3462         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3463         die "QEMU flag querying VM exited with code " . $rc if $rc;
3464
3465         eval {
3466             my $cmd_result = mon_cmd(
3467                 $fakevmid,
3468                 'query-cpu-model-expansion',
3469                 type => 'full',
3470                 model => { name => 'host' }
3471             );
3472
3473             my $props = $cmd_result->{model}->{props};
3474             foreach my $prop (keys %$props) {
3475                 next if $props->{$prop} ne '1';
3476                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3477                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3478                 # We only keep those with underscores, to match /proc/cpuinfo
3479                 $prop =~ s/\.|-/_/g;
3480                 $flags->{$prop} = 1;
3481             }
3482         };
3483         my $err = $@;
3484
3485         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3486         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3487
3488         die $err if $err;
3489
3490         return [ sort keys %$flags ];
3491     };
3492
3493     # We need to query QEMU twice, since KVM and TCG have different supported flags
3494     PVE::QemuConfig->lock_config($fakevmid, sub {
3495         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3496         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3497
3498         if ($kvm_supported) {
3499             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3500             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3501         }
3502     });
3503
3504     return $flags;
3505 }
3506
3507 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3508 my $understood_cpu_flag_dir = "/usr/share/kvm";
3509 sub query_understood_cpu_flags {
3510     my $arch = get_host_arch();
3511     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3512
3513     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3514         if ! -e $filepath;
3515
3516     my $raw = file_get_contents($filepath);
3517     $raw =~ s/^\s+|\s+$//g;
3518     my @flags = split(/\s+/, $raw);
3519
3520     return \@flags;
3521 }
3522
3523 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3524 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3525 my sub should_disable_smm {
3526     my ($conf, $vga) = @_;
3527
3528     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3529         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3530 }
3531
3532 my sub print_ovmf_drive_commandlines {
3533     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3534
3535     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3536
3537     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3538
3539     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3540     if ($d) {
3541         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3542         my ($path, $format) = $d->@{'file', 'format'};
3543         if ($storeid) {
3544             $path = PVE::Storage::path($storecfg, $d->{file});
3545             if (!defined($format)) {
3546                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3547                 $format = qemu_img_format($scfg, $volname);
3548             }
3549         } elsif (!defined($format)) {
3550             die "efidisk format must be specified\n";
3551         }
3552         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3553         if ($path =~ m/^rbd:/) {
3554             $var_drive_str .= ',cache=writeback';
3555             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3556         }
3557         $var_drive_str .= ",format=$format,file=$path";
3558
3559         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3560         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3561     } else {
3562         log_warn("no efidisk configured! Using temporary efivars disk.");
3563         my $path = "/tmp/$vmid-ovmf.fd";
3564         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3565         $var_drive_str .= ",format=raw,file=$path";
3566         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3567     }
3568
3569     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3570 }
3571
3572 sub config_to_command {
3573     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3574         $pbs_backing) = @_;
3575
3576     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3577     my $devices = [];
3578     my $bridges = {};
3579     my $ostype = $conf->{ostype};
3580     my $winversion = windows_version($ostype);
3581     my $kvm = $conf->{kvm};
3582     my $nodename = nodename();
3583
3584     my $arch = get_vm_arch($conf);
3585     my $kvm_binary = get_command_for_arch($arch);
3586     my $kvmver = kvm_user_version($kvm_binary);
3587
3588     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3589         $kvmver //= "undefined";
3590         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3591     }
3592
3593     my $add_pve_version = min_version($kvmver, 4, 1);
3594
3595     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3596     my $machine_version = extract_version($machine_type, $kvmver);
3597     $kvm //= 1 if is_native($arch);
3598
3599     $machine_version =~ m/(\d+)\.(\d+)/;
3600     my ($machine_major, $machine_minor) = ($1, $2);
3601
3602     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3603         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3604     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3605         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3606             ." please upgrade node '$nodename'\n"
3607     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3608         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3609         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3610             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3611             ." node '$nodename'\n";
3612     }
3613
3614     # if a specific +pve version is required for a feature, use $version_guard
3615     # instead of min_version to allow machines to be run with the minimum
3616     # required version
3617     my $required_pve_version = 0;
3618     my $version_guard = sub {
3619         my ($major, $minor, $pve) = @_;
3620         return 0 if !min_version($machine_version, $major, $minor, $pve);
3621         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3622         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3623         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3624         return 1;
3625     };
3626
3627     if ($kvm && !defined kvm_version()) {
3628         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3629             ." or enable in BIOS.\n";
3630     }
3631
3632     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3633     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3634     my $use_old_bios_files = undef;
3635     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3636
3637     my $cmd = [];
3638     if ($conf->{affinity}) {
3639         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3640     }
3641
3642     push @$cmd, $kvm_binary;
3643
3644     push @$cmd, '-id', $vmid;
3645
3646     my $vmname = $conf->{name} || "vm$vmid";
3647
3648     push @$cmd, '-name', "$vmname,debug-threads=on";
3649
3650     push @$cmd, '-no-shutdown';
3651
3652     my $use_virtio = 0;
3653
3654     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3655     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3656     push @$cmd, '-mon', "chardev=qmp,mode=control";
3657
3658     if (min_version($machine_version, 2, 12)) {
3659         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3660         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3661     }
3662
3663     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3664
3665     push @$cmd, '-daemonize';
3666
3667     if ($conf->{smbios1}) {
3668         my $smbios_conf = parse_smbios1($conf->{smbios1});
3669         if ($smbios_conf->{base64}) {
3670             # Do not pass base64 flag to qemu
3671             delete $smbios_conf->{base64};
3672             my $smbios_string = "";
3673             foreach my $key (keys %$smbios_conf) {
3674                 my $value;
3675                 if ($key eq "uuid") {
3676                     $value = $smbios_conf->{uuid}
3677                 } else {
3678                     $value = decode_base64($smbios_conf->{$key});
3679                 }
3680                 # qemu accepts any binary data, only commas need escaping by double comma
3681                 $value =~ s/,/,,/g;
3682                 $smbios_string .= "," . $key . "=" . $value if $value;
3683             }
3684             push @$cmd, '-smbios', "type=1" . $smbios_string;
3685         } else {
3686             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3687         }
3688     }
3689
3690     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3691         my ($code_drive_str, $var_drive_str) =
3692             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3693         push $cmd->@*, '-drive', $code_drive_str;
3694         push $cmd->@*, '-drive', $var_drive_str;
3695     }
3696
3697     if ($q35) { # tell QEMU to load q35 config early
3698         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3699         if (min_version($machine_version, 4, 0)) {
3700             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3701         } else {
3702             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3703         }
3704     }
3705
3706     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3707         push @$cmd, $fixups->@*;
3708     }
3709
3710     if ($conf->{vmgenid}) {
3711         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3712     }
3713
3714     # add usb controllers
3715     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3716         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3717     push @$devices, @usbcontrollers if @usbcontrollers;
3718     my $vga = parse_vga($conf->{vga});
3719
3720     my $qxlnum = vga_conf_has_spice($conf->{vga});
3721     $vga->{type} = 'qxl' if $qxlnum;
3722
3723     if (!$vga->{type}) {
3724         if ($arch eq 'aarch64') {
3725             $vga->{type} = 'virtio';
3726         } elsif (min_version($machine_version, 2, 9)) {
3727             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3728         } else {
3729             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3730         }
3731     }
3732
3733     # enable absolute mouse coordinates (needed by vnc)
3734     my $tablet = $conf->{tablet};
3735     if (!defined($tablet)) {
3736         $tablet = $defaults->{tablet};
3737         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3738         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3739     }
3740
3741     if ($tablet) {
3742         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3743         my $kbd = print_keyboarddevice_full($conf, $arch);
3744         push @$devices, '-device', $kbd if defined($kbd);
3745     }
3746
3747     my $bootorder = device_bootorder($conf);
3748
3749     # host pci device passthrough
3750     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3751         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3752
3753     # usb devices
3754     my $usb_dev_features = {};
3755     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3756
3757     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3758         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3759     push @$devices, @usbdevices if @usbdevices;
3760
3761     # serial devices
3762     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3763         my $path = $conf->{"serial$i"} or next;
3764         if ($path eq 'socket') {
3765             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3766             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3767             # On aarch64, serial0 is the UART device. QEMU only allows
3768             # connecting UART devices via the '-serial' command line, as
3769             # the device has a fixed slot on the hardware...
3770             if ($arch eq 'aarch64' && $i == 0) {
3771                 push @$devices, '-serial', "chardev:serial$i";
3772             } else {
3773                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3774             }
3775         } else {
3776             die "no such serial device\n" if ! -c $path;
3777             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3778             push @$devices, '-device', "isa-serial,chardev=serial$i";
3779         }
3780     }
3781
3782     # parallel devices
3783     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3784         if (my $path = $conf->{"parallel$i"}) {
3785             die "no such parallel device\n" if ! -c $path;
3786             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3787             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3788             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3789         }
3790     }
3791
3792     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3793         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3794         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3795         push @$devices, @$audio_devs;
3796     }
3797
3798     add_tpm_device($vmid, $devices, $conf);
3799
3800     my $sockets = 1;
3801     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3802     $sockets = $conf->{sockets} if  $conf->{sockets};
3803
3804     my $cores = $conf->{cores} || 1;
3805
3806     my $maxcpus = $sockets * $cores;
3807
3808     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3809
3810     my $allowed_vcpus = $cpuinfo->{cpus};
3811
3812     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3813
3814     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3815         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3816         for (my $i = 2; $i <= $vcpus; $i++)  {
3817             my $cpustr = print_cpu_device($conf,$i);
3818             push @$cmd, '-device', $cpustr;
3819         }
3820
3821     } else {
3822
3823         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3824     }
3825     push @$cmd, '-nodefaults';
3826
3827     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3828
3829     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3830
3831     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3832
3833     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3834         push @$devices, '-device', print_vga_device(
3835             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3836
3837         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3838
3839         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3840         push @$cmd,  '-vnc', "unix:$socket,password=on";
3841     } else {
3842         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3843         push @$cmd, '-nographic';
3844     }
3845
3846     # time drift fix
3847     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3848     my $useLocaltime = $conf->{localtime};
3849
3850     if ($winversion >= 5) { # windows
3851         $useLocaltime = 1 if !defined($conf->{localtime});
3852
3853         # use time drift fix when acpi is enabled
3854         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3855             $tdf = 1 if !defined($conf->{tdf});
3856         }
3857     }
3858
3859     if ($winversion >= 6) {
3860         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3861         push @$cmd, '-no-hpet';
3862     }
3863
3864     push @$rtcFlags, 'driftfix=slew' if $tdf;
3865
3866     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3867         push @$rtcFlags, "base=$conf->{startdate}";
3868     } elsif ($useLocaltime) {
3869         push @$rtcFlags, 'base=localtime';
3870     }
3871
3872     if ($forcecpu) {
3873         push @$cmd, '-cpu', $forcecpu;
3874     } else {
3875         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3876     }
3877
3878     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3879
3880     push @$cmd, '-S' if $conf->{freeze};
3881
3882     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3883
3884     my $guest_agent = parse_guest_agent($conf);
3885
3886     if ($guest_agent->{enabled}) {
3887         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3888         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3889
3890         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3891             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3892             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3893             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3894         } elsif ($guest_agent->{type} eq 'isa') {
3895             push @$devices, '-device', "isa-serial,chardev=qga0";
3896         }
3897     }
3898
3899     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3900     if ($rng && $version_guard->(4, 1, 2)) {
3901         check_rng_source($rng->{source});
3902
3903         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3904         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3905         my $limiter_str = "";
3906         if ($max_bytes) {
3907             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3908         }
3909
3910         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3911         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3912         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3913     }
3914
3915     my $spice_port;
3916
3917     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3918         if ($qxlnum > 1) {
3919             if ($winversion){
3920                 for (my $i = 1; $i < $qxlnum; $i++){
3921                     push @$devices, '-device', print_vga_device(
3922                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3923                 }
3924             } else {
3925                 # assume other OS works like Linux
3926                 my ($ram, $vram) = ("134217728", "67108864");
3927                 if ($vga->{memory}) {
3928                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3929                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3930                 }
3931                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3932                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3933             }
3934         }
3935
3936         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3937
3938         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3939         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3940         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3941
3942         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3943         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3944         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3945
3946         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3947         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3948
3949         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3950         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3951         if ($spice_enhancement->{foldersharing}) {
3952             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3953             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3954         }
3955
3956         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3957         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3958             if $spice_enhancement->{videostreaming};
3959
3960         push @$devices, '-spice', "$spice_opts";
3961     }
3962
3963     # enable balloon by default, unless explicitly disabled
3964     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3965         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3966         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3967         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3968         push @$devices, '-device', $ballooncmd;
3969     }
3970
3971     if ($conf->{watchdog}) {
3972         my $wdopts = parse_watchdog($conf->{watchdog});
3973         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3974         my $watchdog = $wdopts->{model} || 'i6300esb';
3975         push @$devices, '-device', "$watchdog$pciaddr";
3976         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3977     }
3978
3979     my $vollist = [];
3980     my $scsicontroller = {};
3981     my $ahcicontroller = {};
3982     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3983
3984     # Add iscsi initiator name if available
3985     if (my $initiator = get_initiator_name()) {
3986         push @$devices, '-iscsi', "initiator-name=$initiator";
3987     }
3988
3989     PVE::QemuConfig->foreach_volume($conf, sub {
3990         my ($ds, $drive) = @_;
3991
3992         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3993             check_volume_storage_type($storecfg, $drive->{file});
3994             push @$vollist, $drive->{file};
3995         }
3996
3997         # ignore efidisk here, already added in bios/fw handling code above
3998         return if $drive->{interface} eq 'efidisk';
3999         # similar for TPM
4000         return if $drive->{interface} eq 'tpmstate';
4001
4002         $use_virtio = 1 if $ds =~ m/^virtio/;
4003
4004         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4005
4006         if ($drive->{interface} eq 'virtio'){
4007            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4008         }
4009
4010         if ($drive->{interface} eq 'scsi') {
4011
4012             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4013
4014             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4015                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4016
4017             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4018             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4019
4020             my $iothread = '';
4021             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4022                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4023                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4024             } elsif ($drive->{iothread}) {
4025                 log_warn(
4026                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4027                 );
4028             }
4029
4030             my $queues = '';
4031             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4032                 $queues = ",num_queues=$drive->{queues}";
4033             }
4034
4035             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4036                 if !$scsicontroller->{$controller};
4037             $scsicontroller->{$controller}=1;
4038         }
4039
4040         if ($drive->{interface} eq 'sata') {
4041             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4042             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4043             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4044                 if !$ahcicontroller->{$controller};
4045             $ahcicontroller->{$controller}=1;
4046         }
4047
4048         my $pbs_conf = $pbs_backing->{$ds};
4049         my $pbs_name = undef;
4050         if ($pbs_conf) {
4051             $pbs_name = "drive-$ds-pbs";
4052             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4053         }
4054
4055         my $drive_cmd = print_drive_commandline_full(
4056             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4057
4058         # extra protection for templates, but SATA and IDE don't support it..
4059         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4060
4061         push @$devices, '-drive',$drive_cmd;
4062         push @$devices, '-device', print_drivedevice_full(
4063             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4064     });
4065
4066     for (my $i = 0; $i < $MAX_NETS; $i++) {
4067         my $netname = "net$i";
4068
4069         next if !$conf->{$netname};
4070         my $d = parse_net($conf->{$netname});
4071         next if !$d;
4072         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4073
4074         $use_virtio = 1 if $d->{model} eq 'virtio';
4075
4076         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4077
4078         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4079         push @$devices, '-netdev', $netdevfull;
4080
4081         my $netdevicefull = print_netdevice_full(
4082             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4083
4084         push @$devices, '-device', $netdevicefull;
4085     }
4086
4087     if ($conf->{ivshmem}) {
4088         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4089
4090         my $bus;
4091         if ($q35) {
4092             $bus = print_pcie_addr("ivshmem");
4093         } else {
4094             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4095         }
4096
4097         my $ivshmem_name = $ivshmem->{name} // $vmid;
4098         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4099
4100         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4101         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4102             .",size=$ivshmem->{size}M";
4103     }
4104
4105     # pci.4 is nested in pci.1
4106     $bridges->{1} = 1 if $bridges->{4};
4107
4108     if (!$q35) { # add pci bridges
4109         if (min_version($machine_version, 2, 3)) {
4110            $bridges->{1} = 1;
4111            $bridges->{2} = 1;
4112         }
4113         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4114     }
4115
4116     for my $k (sort {$b cmp $a} keys %$bridges) {
4117         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4118
4119         my $k_name = $k;
4120         if ($k == 2 && $legacy_igd) {
4121             $k_name = "$k-igd";
4122         }
4123         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4124         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4125
4126         if ($q35) { # add after -readconfig pve-q35.cfg
4127             splice @$devices, 2, 0, '-device', $devstr;
4128         } else {
4129             unshift @$devices, '-device', $devstr if $k > 0;
4130         }
4131     }
4132
4133     if (!$kvm) {
4134         push @$machineFlags, 'accel=tcg';
4135     }
4136
4137     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4138
4139     my $machine_type_min = $machine_type;
4140     if ($add_pve_version) {
4141         $machine_type_min =~ s/\+pve\d+$//;
4142         $machine_type_min .= "+pve$required_pve_version";
4143     }
4144     push @$machineFlags, "type=${machine_type_min}";
4145
4146     push @$cmd, @$devices;
4147     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4148     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4149     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4150
4151     if (my $vmstate = $conf->{vmstate}) {
4152         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4153         push @$vollist, $vmstate;
4154         push @$cmd, '-loadstate', $statepath;
4155         print "activating and using '$vmstate' as vmstate\n";
4156     }
4157
4158     if (PVE::QemuConfig->is_template($conf)) {
4159         # needed to workaround base volumes being read-only
4160         push @$cmd, '-snapshot';
4161     }
4162
4163     # add custom args
4164     if ($conf->{args}) {
4165         my $aa = PVE::Tools::split_args($conf->{args});
4166         push @$cmd, @$aa;
4167     }
4168
4169     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4170 }
4171
4172 sub check_rng_source {
4173     my ($source) = @_;
4174
4175     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4176     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4177         if ! -e $source;
4178
4179     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4180     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4181         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4182         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4183         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4184             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4185             ." to the host.\n";
4186     }
4187 }
4188
4189 sub spice_port {
4190     my ($vmid) = @_;
4191
4192     my $res = mon_cmd($vmid, 'query-spice');
4193
4194     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4195 }
4196
4197 sub vm_devices_list {
4198     my ($vmid) = @_;
4199
4200     my $res = mon_cmd($vmid, 'query-pci');
4201     my $devices_to_check = [];
4202     my $devices = {};
4203     foreach my $pcibus (@$res) {
4204         push @$devices_to_check, @{$pcibus->{devices}},
4205     }
4206
4207     while (@$devices_to_check) {
4208         my $to_check = [];
4209         for my $d (@$devices_to_check) {
4210             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4211             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4212
4213             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4214             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4215         }
4216         $devices_to_check = $to_check;
4217     }
4218
4219     my $resblock = mon_cmd($vmid, 'query-block');
4220     foreach my $block (@$resblock) {
4221         if($block->{device} =~ m/^drive-(\S+)/){
4222                 $devices->{$1} = 1;
4223         }
4224     }
4225
4226     my $resmice = mon_cmd($vmid, 'query-mice');
4227     foreach my $mice (@$resmice) {
4228         if ($mice->{name} eq 'QEMU HID Tablet') {
4229             $devices->{tablet} = 1;
4230             last;
4231         }
4232     }
4233
4234     # for usb devices there is no query-usb
4235     # but we can iterate over the entries in
4236     # qom-list path=/machine/peripheral
4237     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4238     foreach my $per (@$resperipheral) {
4239         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4240             $devices->{$per->{name}} = 1;
4241         }
4242     }
4243
4244     return $devices;
4245 }
4246
4247 sub vm_deviceplug {
4248     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4249
4250     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4251
4252     my $devices_list = vm_devices_list($vmid);
4253     return 1 if defined($devices_list->{$deviceid});
4254
4255     # add PCI bridge if we need it for the device
4256     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4257
4258     if ($deviceid eq 'tablet') {
4259         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4260     } elsif ($deviceid eq 'keyboard') {
4261         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4262     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4263         my $id = $1;
4264         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4265         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4266     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4267         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4268     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4269         qemu_iothread_add($vmid, $deviceid, $device);
4270
4271         qemu_driveadd($storecfg, $vmid, $device);
4272         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4273
4274         qemu_deviceadd($vmid, $devicefull);
4275         eval { qemu_deviceaddverify($vmid, $deviceid); };
4276         if (my $err = $@) {
4277             eval { qemu_drivedel($vmid, $deviceid); };
4278             warn $@ if $@;
4279             die $err;
4280         }
4281     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4282         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4283         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4284         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4285
4286         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4287
4288         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4289             qemu_iothread_add($vmid, $deviceid, $device);
4290             $devicefull .= ",iothread=iothread-$deviceid";
4291         }
4292
4293         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4294             $devicefull .= ",num_queues=$device->{queues}";
4295         }
4296
4297         qemu_deviceadd($vmid, $devicefull);
4298         qemu_deviceaddverify($vmid, $deviceid);
4299     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4300         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4301         qemu_driveadd($storecfg, $vmid, $device);
4302
4303         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4304         eval { qemu_deviceadd($vmid, $devicefull); };
4305         if (my $err = $@) {
4306             eval { qemu_drivedel($vmid, $deviceid); };
4307             warn $@ if $@;
4308             die $err;
4309         }
4310     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4311         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4312
4313         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4314         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4315         my $use_old_bios_files = undef;
4316         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4317
4318         my $netdevicefull = print_netdevice_full(
4319             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4320         qemu_deviceadd($vmid, $netdevicefull);
4321         eval {
4322             qemu_deviceaddverify($vmid, $deviceid);
4323             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4324         };
4325         if (my $err = $@) {
4326             eval { qemu_netdevdel($vmid, $deviceid); };
4327             warn $@ if $@;
4328             die $err;
4329         }
4330     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4331         my $bridgeid = $2;
4332         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4333         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4334
4335         qemu_deviceadd($vmid, $devicefull);
4336         qemu_deviceaddverify($vmid, $deviceid);
4337     } else {
4338         die "can't hotplug device '$deviceid'\n";
4339     }
4340
4341     return 1;
4342 }
4343
4344 # fixme: this should raise exceptions on error!
4345 sub vm_deviceunplug {
4346     my ($vmid, $conf, $deviceid) = @_;
4347
4348     my $devices_list = vm_devices_list($vmid);
4349     return 1 if !defined($devices_list->{$deviceid});
4350
4351     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4352     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4353
4354     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4355         qemu_devicedel($vmid, $deviceid);
4356     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4357         qemu_devicedel($vmid, $deviceid);
4358         qemu_devicedelverify($vmid, $deviceid);
4359     } elsif ($deviceid =~ m/^usb\d+$/) {
4360         qemu_devicedel($vmid, $deviceid);
4361         qemu_devicedelverify($vmid, $deviceid);
4362     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4363         my $device = parse_drive($deviceid, $conf->{$deviceid});
4364
4365         qemu_devicedel($vmid, $deviceid);
4366         qemu_devicedelverify($vmid, $deviceid);
4367         qemu_drivedel($vmid, $deviceid);
4368         qemu_iothread_del($vmid, $deviceid, $device);
4369     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4370         qemu_devicedel($vmid, $deviceid);
4371         qemu_devicedelverify($vmid, $deviceid);
4372     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4373         my $device = parse_drive($deviceid, $conf->{$deviceid});
4374
4375         qemu_devicedel($vmid, $deviceid);
4376         qemu_devicedelverify($vmid, $deviceid);
4377         qemu_drivedel($vmid, $deviceid);
4378         qemu_deletescsihw($conf, $vmid, $deviceid);
4379
4380         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4381             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4382     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4383         qemu_devicedel($vmid, $deviceid);
4384         qemu_devicedelverify($vmid, $deviceid);
4385         qemu_netdevdel($vmid, $deviceid);
4386     } else {
4387         die "can't unplug device '$deviceid'\n";
4388     }
4389
4390     return 1;
4391 }
4392
4393 sub qemu_spice_usbredir_chardev_add {
4394     my ($vmid, $id) = @_;
4395
4396     mon_cmd($vmid, "chardev-add" , (
4397         id => $id,
4398         backend => {
4399             type => 'spicevmc',
4400             data => {
4401                 type => "usbredir",
4402             },
4403         },
4404     ));
4405 }
4406
4407 sub qemu_deviceadd {
4408     my ($vmid, $devicefull) = @_;
4409
4410     $devicefull = "driver=".$devicefull;
4411     my %options =  split(/[=,]/, $devicefull);
4412
4413     mon_cmd($vmid, "device_add" , %options);
4414 }
4415
4416 sub qemu_devicedel {
4417     my ($vmid, $deviceid) = @_;
4418
4419     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4420 }
4421
4422 sub qemu_iothread_add {
4423     my ($vmid, $deviceid, $device) = @_;
4424
4425     if ($device->{iothread}) {
4426         my $iothreads = vm_iothreads_list($vmid);
4427         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4428     }
4429 }
4430
4431 sub qemu_iothread_del {
4432     my ($vmid, $deviceid, $device) = @_;
4433
4434     if ($device->{iothread}) {
4435         my $iothreads = vm_iothreads_list($vmid);
4436         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4437     }
4438 }
4439
4440 sub qemu_objectadd {
4441     my ($vmid, $objectid, $qomtype) = @_;
4442
4443     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4444
4445     return 1;
4446 }
4447
4448 sub qemu_objectdel {
4449     my ($vmid, $objectid) = @_;
4450
4451     mon_cmd($vmid, "object-del", id => $objectid);
4452
4453     return 1;
4454 }
4455
4456 sub qemu_driveadd {
4457     my ($storecfg, $vmid, $device) = @_;
4458
4459     my $kvmver = get_running_qemu_version($vmid);
4460     my $io_uring = min_version($kvmver, 6, 0);
4461     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4462     $drive =~ s/\\/\\\\/g;
4463     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4464
4465     # If the command succeeds qemu prints: "OK"
4466     return 1 if $ret =~ m/OK/s;
4467
4468     die "adding drive failed: $ret\n";
4469 }
4470
4471 sub qemu_drivedel {
4472     my ($vmid, $deviceid) = @_;
4473
4474     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4475     $ret =~ s/^\s+//;
4476
4477     return 1 if $ret eq "";
4478
4479     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4480     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4481
4482     die "deleting drive $deviceid failed : $ret\n";
4483 }
4484
4485 sub qemu_deviceaddverify {
4486     my ($vmid, $deviceid) = @_;
4487
4488     for (my $i = 0; $i <= 5; $i++) {
4489          my $devices_list = vm_devices_list($vmid);
4490          return 1 if defined($devices_list->{$deviceid});
4491          sleep 1;
4492     }
4493
4494     die "error on hotplug device '$deviceid'\n";
4495 }
4496
4497
4498 sub qemu_devicedelverify {
4499     my ($vmid, $deviceid) = @_;
4500
4501     # need to verify that the device is correctly removed as device_del
4502     # is async and empty return is not reliable
4503
4504     for (my $i = 0; $i <= 5; $i++) {
4505          my $devices_list = vm_devices_list($vmid);
4506          return 1 if !defined($devices_list->{$deviceid});
4507          sleep 1;
4508     }
4509
4510     die "error on hot-unplugging device '$deviceid'\n";
4511 }
4512
4513 sub qemu_findorcreatescsihw {
4514     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4515
4516     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4517
4518     my $scsihwid="$controller_prefix$controller";
4519     my $devices_list = vm_devices_list($vmid);
4520
4521     if (!defined($devices_list->{$scsihwid})) {
4522         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4523     }
4524
4525     return 1;
4526 }
4527
4528 sub qemu_deletescsihw {
4529     my ($conf, $vmid, $opt) = @_;
4530
4531     my $device = parse_drive($opt, $conf->{$opt});
4532
4533     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4534         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4535         return 1;
4536     }
4537
4538     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4539
4540     my $devices_list = vm_devices_list($vmid);
4541     foreach my $opt (keys %{$devices_list}) {
4542         if (is_valid_drivename($opt)) {
4543             my $drive = parse_drive($opt, $conf->{$opt});
4544             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4545                 return 1;
4546             }
4547         }
4548     }
4549
4550     my $scsihwid="scsihw$controller";
4551
4552     vm_deviceunplug($vmid, $conf, $scsihwid);
4553
4554     return 1;
4555 }
4556
4557 sub qemu_add_pci_bridge {
4558     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4559
4560     my $bridges = {};
4561
4562     my $bridgeid;
4563
4564     print_pci_addr($device, $bridges, $arch, $machine_type);
4565
4566     while (my ($k, $v) = each %$bridges) {
4567         $bridgeid = $k;
4568     }
4569     return 1 if !defined($bridgeid) || $bridgeid < 1;
4570
4571     my $bridge = "pci.$bridgeid";
4572     my $devices_list = vm_devices_list($vmid);
4573
4574     if (!defined($devices_list->{$bridge})) {
4575         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4576     }
4577
4578     return 1;
4579 }
4580
4581 sub qemu_set_link_status {
4582     my ($vmid, $device, $up) = @_;
4583
4584     mon_cmd($vmid, "set_link", name => $device,
4585                up => $up ? JSON::true : JSON::false);
4586 }
4587
4588 sub qemu_netdevadd {
4589     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4590
4591     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4592     my %options =  split(/[=,]/, $netdev);
4593
4594     if (defined(my $vhost = $options{vhost})) {
4595         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4596     }
4597
4598     if (defined(my $queues = $options{queues})) {
4599         $options{queues} = $queues + 0;
4600     }
4601
4602     mon_cmd($vmid, "netdev_add",  %options);
4603     return 1;
4604 }
4605
4606 sub qemu_netdevdel {
4607     my ($vmid, $deviceid) = @_;
4608
4609     mon_cmd($vmid, "netdev_del", id => $deviceid);
4610 }
4611
4612 sub qemu_usb_hotplug {
4613     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4614
4615     return if !$device;
4616
4617     # remove the old one first
4618     vm_deviceunplug($vmid, $conf, $deviceid);
4619
4620     # check if xhci controller is necessary and available
4621     my $devicelist = vm_devices_list($vmid);
4622
4623     if (!$devicelist->{xhci}) {
4624         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4625         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4626     }
4627
4628     # print_usbdevice_full expects the parsed device
4629     my $d = parse_usb_device($device->{host});
4630     $d->{usb3} = $device->{usb3};
4631
4632     # add the new one
4633     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4634 }
4635
4636 sub qemu_cpu_hotplug {
4637     my ($vmid, $conf, $vcpus) = @_;
4638
4639     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4640
4641     my $sockets = 1;
4642     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4643     $sockets = $conf->{sockets} if  $conf->{sockets};
4644     my $cores = $conf->{cores} || 1;
4645     my $maxcpus = $sockets * $cores;
4646
4647     $vcpus = $maxcpus if !$vcpus;
4648
4649     die "you can't add more vcpus than maxcpus\n"
4650         if $vcpus > $maxcpus;
4651
4652     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4653
4654     if ($vcpus < $currentvcpus) {
4655
4656         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4657
4658             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4659                 qemu_devicedel($vmid, "cpu$i");
4660                 my $retry = 0;
4661                 my $currentrunningvcpus = undef;
4662                 while (1) {
4663                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4664                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4665                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4666                     $retry++;
4667                     sleep 1;
4668                 }
4669                 #update conf after each succesfull cpu unplug
4670                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4671                 PVE::QemuConfig->write_config($vmid, $conf);
4672             }
4673         } else {
4674             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4675         }
4676
4677         return;
4678     }
4679
4680     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4681     die "vcpus in running vm does not match its configuration\n"
4682         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4683
4684     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4685
4686         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4687             my $cpustr = print_cpu_device($conf, $i);
4688             qemu_deviceadd($vmid, $cpustr);
4689
4690             my $retry = 0;
4691             my $currentrunningvcpus = undef;
4692             while (1) {
4693                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4694                 last if scalar(@{$currentrunningvcpus}) == $i;
4695                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4696                 sleep 1;
4697                 $retry++;
4698             }
4699             #update conf after each succesfull cpu hotplug
4700             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4701             PVE::QemuConfig->write_config($vmid, $conf);
4702         }
4703     } else {
4704
4705         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4706             mon_cmd($vmid, "cpu-add", id => int($i));
4707         }
4708     }
4709 }
4710
4711 sub qemu_block_set_io_throttle {
4712     my ($vmid, $deviceid,
4713         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4714         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4715         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4716         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4717
4718     return if !check_running($vmid) ;
4719
4720     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4721         bps => int($bps),
4722         bps_rd => int($bps_rd),
4723         bps_wr => int($bps_wr),
4724         iops => int($iops),
4725         iops_rd => int($iops_rd),
4726         iops_wr => int($iops_wr),
4727         bps_max => int($bps_max),
4728         bps_rd_max => int($bps_rd_max),
4729         bps_wr_max => int($bps_wr_max),
4730         iops_max => int($iops_max),
4731         iops_rd_max => int($iops_rd_max),
4732         iops_wr_max => int($iops_wr_max),
4733         bps_max_length => int($bps_max_length),
4734         bps_rd_max_length => int($bps_rd_max_length),
4735         bps_wr_max_length => int($bps_wr_max_length),
4736         iops_max_length => int($iops_max_length),
4737         iops_rd_max_length => int($iops_rd_max_length),
4738         iops_wr_max_length => int($iops_wr_max_length),
4739     );
4740
4741 }
4742
4743 sub qemu_block_resize {
4744     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4745
4746     my $running = check_running($vmid);
4747
4748     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4749
4750     return if !$running;
4751
4752     my $padding = (1024 - $size % 1024) % 1024;
4753     $size = $size + $padding;
4754
4755     mon_cmd(
4756         $vmid,
4757         "block_resize",
4758         device => $deviceid,
4759         size => int($size),
4760         timeout => 60,
4761     );
4762 }
4763
4764 sub qemu_volume_snapshot {
4765     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4766
4767     my $running = check_running($vmid);
4768
4769     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4770         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4771     } else {
4772         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4773     }
4774 }
4775
4776 sub qemu_volume_snapshot_delete {
4777     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4778
4779     my $running = check_running($vmid);
4780
4781     if($running) {
4782
4783         $running = undef;
4784         my $conf = PVE::QemuConfig->load_config($vmid);
4785         PVE::QemuConfig->foreach_volume($conf, sub {
4786             my ($ds, $drive) = @_;
4787             $running = 1 if $drive->{file} eq $volid;
4788         });
4789     }
4790
4791     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4792         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4793     } else {
4794         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4795     }
4796 }
4797
4798 sub set_migration_caps {
4799     my ($vmid, $savevm) = @_;
4800
4801     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4802
4803     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4804     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4805
4806     my $cap_ref = [];
4807
4808     my $enabled_cap = {
4809         "auto-converge" => 1,
4810         "xbzrle" => 1,
4811         "x-rdma-pin-all" => 0,
4812         "zero-blocks" => 0,
4813         "compress" => 0,
4814         "dirty-bitmaps" => $dirty_bitmaps,
4815     };
4816
4817     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4818
4819     for my $supported_capability (@$supported_capabilities) {
4820         push @$cap_ref, {
4821             capability => $supported_capability->{capability},
4822             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4823         };
4824     }
4825
4826     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4827 }
4828
4829 sub foreach_volid {
4830     my ($conf, $func, @param) = @_;
4831
4832     my $volhash = {};
4833
4834     my $test_volid = sub {
4835         my ($key, $drive, $snapname) = @_;
4836
4837         my $volid = $drive->{file};
4838         return if !$volid;
4839
4840         $volhash->{$volid}->{cdrom} //= 1;
4841         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4842
4843         my $replicate = $drive->{replicate} // 1;
4844         $volhash->{$volid}->{replicate} //= 0;
4845         $volhash->{$volid}->{replicate} = 1 if $replicate;
4846
4847         $volhash->{$volid}->{shared} //= 0;
4848         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4849
4850         $volhash->{$volid}->{referenced_in_config} //= 0;
4851         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4852
4853         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4854             if defined($snapname);
4855
4856         my $size = $drive->{size};
4857         $volhash->{$volid}->{size} //= $size if $size;
4858
4859         $volhash->{$volid}->{is_vmstate} //= 0;
4860         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4861
4862         $volhash->{$volid}->{is_tpmstate} //= 0;
4863         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4864
4865         $volhash->{$volid}->{is_unused} //= 0;
4866         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4867
4868         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4869     };
4870
4871     my $include_opts = {
4872         extra_keys => ['vmstate'],
4873         include_unused => 1,
4874     };
4875
4876     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4877     foreach my $snapname (keys %{$conf->{snapshots}}) {
4878         my $snap = $conf->{snapshots}->{$snapname};
4879         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4880     }
4881
4882     foreach my $volid (keys %$volhash) {
4883         &$func($volid, $volhash->{$volid}, @param);
4884     }
4885 }
4886
4887 my $fast_plug_option = {
4888     'lock' => 1,
4889     'name' => 1,
4890     'onboot' => 1,
4891     'shares' => 1,
4892     'startup' => 1,
4893     'description' => 1,
4894     'protection' => 1,
4895     'vmstatestorage' => 1,
4896     'hookscript' => 1,
4897     'tags' => 1,
4898 };
4899
4900 for my $opt (keys %$confdesc_cloudinit) {
4901     $fast_plug_option->{$opt} = 1;
4902 };
4903
4904 # hotplug changes in [PENDING]
4905 # $selection hash can be used to only apply specified options, for
4906 # example: { cores => 1 } (only apply changed 'cores')
4907 # $errors ref is used to return error messages
4908 sub vmconfig_hotplug_pending {
4909     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4910
4911     my $defaults = load_defaults();
4912     my $arch = get_vm_arch($conf);
4913     my $machine_type = get_vm_machine($conf, undef, $arch);
4914
4915     # commit values which do not have any impact on running VM first
4916     # Note: those option cannot raise errors, we we do not care about
4917     # $selection and always apply them.
4918
4919     my $add_error = sub {
4920         my ($opt, $msg) = @_;
4921         $errors->{$opt} = "hotplug problem - $msg";
4922     };
4923
4924     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4925
4926     my $cloudinit_record_changed = sub {
4927         my ($conf, $opt, $old, $new) = @_;
4928         return if !$cloudinit_pending_properties->{$opt};
4929
4930         my $ci = ($conf->{cloudinit} //= {});
4931
4932         my $recorded = $ci->{$opt};
4933         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4934
4935         if (defined($new)) {
4936             if (defined($old)) {
4937                 # an existing value is being modified
4938                 if (defined($recorded)) {
4939                     # the value was already not in sync
4940                     if ($new eq $recorded) {
4941                         # a value is being reverted to the cloud-init state:
4942                         delete $ci->{$opt};
4943                         delete $added{$opt};
4944                     } else {
4945                         # the value was changed multiple times, do nothing
4946                     }
4947                 } elsif ($added{$opt}) {
4948                     # the value had been marked as added and is being changed, do nothing
4949                 } else {
4950                     # the value is new, record it:
4951                     $ci->{$opt} = $old;
4952                 }
4953             } else {
4954                 # a new value is being added
4955                 if (defined($recorded)) {
4956                     # it was already not in sync
4957                     if ($new eq $recorded) {
4958                         # a value is being reverted to the cloud-init state:
4959                         delete $ci->{$opt};
4960                         delete $added{$opt};
4961                     } else {
4962                         # the value had temporarily been removed, do nothing
4963                     }
4964                 } elsif ($added{$opt}) {
4965                     # the value had been marked as added already, do nothing
4966                 } else {
4967                     # the value is new, add it
4968                     $added{$opt} = 1;
4969                 }
4970             }
4971         } elsif (!defined($old)) {
4972             # a non-existent value is being removed? ignore...
4973         } else {
4974             # a value is being deleted
4975             if (defined($recorded)) {
4976                 # a value was already recorded, just keep it
4977             } elsif ($added{$opt}) {
4978                 # the value was marked as added, remove it
4979                 delete $added{$opt};
4980             } else {
4981                 # a previously unrecorded value is being removed, record the old value:
4982                 $ci->{$opt} = $old;
4983             }
4984         }
4985
4986         my $added = join(',', sort keys %added);
4987         $ci->{added} = $added if length($added);
4988     };
4989
4990     my $changes = 0;
4991     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4992         if ($fast_plug_option->{$opt}) {
4993             my $new = delete $conf->{pending}->{$opt};
4994             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4995             $conf->{$opt} = $new;
4996             $changes = 1;
4997         }
4998     }
4999
5000     if ($changes) {
5001         PVE::QemuConfig->write_config($vmid, $conf);
5002     }
5003
5004     my $ostype = $conf->{ostype};
5005     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5006     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5007     my $usb_hotplug = $hotplug_features->{usb}
5008         && min_version($version, 7, 1)
5009         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5010
5011     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5012     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5013
5014     foreach my $opt (sort keys %$pending_delete_hash) {
5015         next if $selection && !$selection->{$opt};
5016         my $force = $pending_delete_hash->{$opt}->{force};
5017         eval {
5018             if ($opt eq 'hotplug') {
5019                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5020             } elsif ($opt eq 'tablet') {
5021                 die "skip\n" if !$hotplug_features->{usb};
5022                 if ($defaults->{tablet}) {
5023                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5024                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5025                         if $arch eq 'aarch64';
5026                 } else {
5027                     vm_deviceunplug($vmid, $conf, 'tablet');
5028                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5029                 }
5030             } elsif ($opt =~ m/^usb(\d+)$/) {
5031                 my $index = $1;
5032                 die "skip\n" if !$usb_hotplug;
5033                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5034                 vm_deviceunplug($vmid, $conf, $opt);
5035             } elsif ($opt eq 'vcpus') {
5036                 die "skip\n" if !$hotplug_features->{cpu};
5037                 qemu_cpu_hotplug($vmid, $conf, undef);
5038             } elsif ($opt eq 'balloon') {
5039                 # enable balloon device is not hotpluggable
5040                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5041                 # here we reset the ballooning value to memory
5042                 my $balloon = $conf->{memory} || $defaults->{memory};
5043                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5044             } elsif ($fast_plug_option->{$opt}) {
5045                 # do nothing
5046             } elsif ($opt =~ m/^net(\d+)$/) {
5047                 die "skip\n" if !$hotplug_features->{network};
5048                 vm_deviceunplug($vmid, $conf, $opt);
5049             } elsif (is_valid_drivename($opt)) {
5050                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5051                 vm_deviceunplug($vmid, $conf, $opt);
5052                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5053             } elsif ($opt =~ m/^memory$/) {
5054                 die "skip\n" if !$hotplug_features->{memory};
5055                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5056             } elsif ($opt eq 'cpuunits') {
5057                 $cgroup->change_cpu_shares(undef);
5058             } elsif ($opt eq 'cpulimit') {
5059                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5060             } else {
5061                 die "skip\n";
5062             }
5063         };
5064         if (my $err = $@) {
5065             &$add_error($opt, $err) if $err ne "skip\n";
5066         } else {
5067             my $old = delete $conf->{$opt};
5068             $cloudinit_record_changed->($conf, $opt, $old, undef);
5069             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5070         }
5071     }
5072
5073     my $cloudinit_opt;
5074     foreach my $opt (keys %{$conf->{pending}}) {
5075         next if $selection && !$selection->{$opt};
5076         my $value = $conf->{pending}->{$opt};
5077         eval {
5078             if ($opt eq 'hotplug') {
5079                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5080             } elsif ($opt eq 'tablet') {
5081                 die "skip\n" if !$hotplug_features->{usb};
5082                 if ($value == 1) {
5083                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5084                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5085                         if $arch eq 'aarch64';
5086                 } elsif ($value == 0) {
5087                     vm_deviceunplug($vmid, $conf, 'tablet');
5088                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5089                 }
5090             } elsif ($opt =~ m/^usb(\d+)$/) {
5091                 my $index = $1;
5092                 die "skip\n" if !$usb_hotplug;
5093                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5094                 my $id = $opt;
5095                 if ($d->{host} eq 'spice')  {
5096                     $id = "usbredirdev$index";
5097                 }
5098                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5099             } elsif ($opt eq 'vcpus') {
5100                 die "skip\n" if !$hotplug_features->{cpu};
5101                 qemu_cpu_hotplug($vmid, $conf, $value);
5102             } elsif ($opt eq 'balloon') {
5103                 # enable/disable balloning device is not hotpluggable
5104                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5105                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5106                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5107
5108                 # allow manual ballooning if shares is set to zero
5109                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5110                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5111                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5112                 }
5113             } elsif ($opt =~ m/^net(\d+)$/) {
5114                 # some changes can be done without hotplug
5115                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5116                                     $vmid, $opt, $value, $arch, $machine_type);
5117             } elsif (is_valid_drivename($opt)) {
5118                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5119                 # some changes can be done without hotplug
5120                 my $drive = parse_drive($opt, $value);
5121                 if (drive_is_cloudinit($drive)) {
5122                     $cloudinit_opt = [$opt, $drive];
5123                     # apply all the other changes first, then generate the cloudinit disk
5124                     die "skip\n";
5125                 }
5126                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5127                                      $vmid, $opt, $value, $arch, $machine_type);
5128             } elsif ($opt =~ m/^memory$/) { #dimms
5129                 die "skip\n" if !$hotplug_features->{memory};
5130                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5131             } elsif ($opt eq 'cpuunits') {
5132                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5133                 $cgroup->change_cpu_shares($new_cpuunits);
5134             } elsif ($opt eq 'cpulimit') {
5135                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5136                 $cgroup->change_cpu_quota($cpulimit, 100000);
5137             } elsif ($opt eq 'agent') {
5138                 vmconfig_update_agent($conf, $opt, $value);
5139             } else {
5140                 die "skip\n";  # skip non-hot-pluggable options
5141             }
5142         };
5143         if (my $err = $@) {
5144             &$add_error($opt, $err) if $err ne "skip\n";
5145         } else {
5146             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5147             $conf->{$opt} = $value;
5148             delete $conf->{pending}->{$opt};
5149         }
5150     }
5151
5152     if (defined($cloudinit_opt)) {
5153         my ($opt, $drive) = @$cloudinit_opt;
5154         my $value = $conf->{pending}->{$opt};
5155         eval {
5156             my $temp = {%$conf, $opt => $value};
5157             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5158             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5159                                  $vmid, $opt, $value, $arch, $machine_type);
5160         };
5161         if (my $err = $@) {
5162             &$add_error($opt, $err) if $err ne "skip\n";
5163         } else {
5164             $conf->{$opt} = $value;
5165             delete $conf->{pending}->{$opt};
5166         }
5167     }
5168
5169     # unplug xhci controller if no usb device is left
5170     if ($usb_hotplug) {
5171         my $has_usb = 0;
5172         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5173             next if !defined($conf->{"usb$i"});
5174             $has_usb = 1;
5175             last;
5176         }
5177         if (!$has_usb) {
5178             vm_deviceunplug($vmid, $conf, 'xhci');
5179         }
5180     }
5181
5182     PVE::QemuConfig->write_config($vmid, $conf);
5183
5184     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5185         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5186     }
5187 }
5188
5189 sub try_deallocate_drive {
5190     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5191
5192     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5193         my $volid = $drive->{file};
5194         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5195             my $sid = PVE::Storage::parse_volume_id($volid);
5196             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5197
5198             # check if the disk is really unused
5199             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5200                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5201             PVE::Storage::vdisk_free($storecfg, $volid);
5202             return 1;
5203         } else {
5204             # If vm is not owner of this disk remove from config
5205             return 1;
5206         }
5207     }
5208
5209     return;
5210 }
5211
5212 sub vmconfig_delete_or_detach_drive {
5213     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5214
5215     my $drive = parse_drive($opt, $conf->{$opt});
5216
5217     my $rpcenv = PVE::RPCEnvironment::get();
5218     my $authuser = $rpcenv->get_user();
5219
5220     if ($force) {
5221         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5222         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5223     } else {
5224         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5225     }
5226 }
5227
5228
5229
5230 sub vmconfig_apply_pending {
5231     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5232
5233     return if !scalar(keys %{$conf->{pending}});
5234
5235     my $add_apply_error = sub {
5236         my ($opt, $msg) = @_;
5237         my $err_msg = "unable to apply pending change $opt : $msg";
5238         $errors->{$opt} = $err_msg;
5239         warn $err_msg;
5240     };
5241
5242     # cold plug
5243
5244     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5245     foreach my $opt (sort keys %$pending_delete_hash) {
5246         my $force = $pending_delete_hash->{$opt}->{force};
5247         eval {
5248             if ($opt =~ m/^unused/) {
5249                 die "internal error";
5250             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5251                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5252             }
5253         };
5254         if (my $err = $@) {
5255             $add_apply_error->($opt, $err);
5256         } else {
5257             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5258             delete $conf->{$opt};
5259         }
5260     }
5261
5262     PVE::QemuConfig->cleanup_pending($conf);
5263
5264     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5265
5266     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5267         next if $opt eq 'delete'; # just to be sure
5268         eval {
5269             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5270                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5271             }
5272         };
5273         if (my $err = $@) {
5274             $add_apply_error->($opt, $err);
5275         } else {
5276
5277             if (is_valid_drivename($opt)) {
5278                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5279                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5280             }
5281
5282             $conf->{$opt} = delete $conf->{pending}->{$opt};
5283         }
5284     }
5285
5286     # write all changes at once to avoid unnecessary i/o
5287     PVE::QemuConfig->write_config($vmid, $conf);
5288     if ($generate_cloudinit) {
5289         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5290             # After successful generation and if there were changes to be applied, update the
5291             # config to drop the {cloudinit} entry.
5292             PVE::QemuConfig->write_config($vmid, $conf);
5293         }
5294     }
5295 }
5296
5297 sub vmconfig_update_net {
5298     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5299
5300     my $newnet = parse_net($value);
5301
5302     if ($conf->{$opt}) {
5303         my $oldnet = parse_net($conf->{$opt});
5304
5305         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5306             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5307             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5308             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5309
5310             # for non online change, we try to hot-unplug
5311             die "skip\n" if !$hotplug;
5312             vm_deviceunplug($vmid, $conf, $opt);
5313         } else {
5314
5315             die "internal error" if $opt !~ m/net(\d+)/;
5316             my $iface = "tap${vmid}i$1";
5317
5318             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5319                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5320                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5321                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5322                 PVE::Network::tap_unplug($iface);
5323
5324                 if ($have_sdn) {
5325                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5326                 } else {
5327                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5328                 }
5329             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5330                 # Rate can be applied on its own but any change above needs to
5331                 # include the rate in tap_plug since OVS resets everything.
5332                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5333             }
5334
5335             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5336                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5337             }
5338
5339             return 1;
5340         }
5341     }
5342
5343     if ($hotplug) {
5344         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5345     } else {
5346         die "skip\n";
5347     }
5348 }
5349
5350 sub vmconfig_update_agent {
5351     my ($conf, $opt, $value) = @_;
5352
5353     die "skip\n" if !$conf->{$opt};
5354
5355     my $hotplug_options = { fstrim_cloned_disks => 1 };
5356
5357     my $old_agent = parse_guest_agent($conf);
5358     my $agent = parse_guest_agent({$opt => $value});
5359
5360     for my $option (keys %$agent) { # added/changed options
5361         next if defined($hotplug_options->{$option});
5362         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5363     }
5364
5365     for my $option (keys %$old_agent) { # removed options
5366         next if defined($hotplug_options->{$option});
5367         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5368     }
5369
5370     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5371 }
5372
5373 sub vmconfig_update_disk {
5374     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5375
5376     my $drive = parse_drive($opt, $value);
5377
5378     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5379         my $media = $drive->{media} || 'disk';
5380         my $oldmedia = $old_drive->{media} || 'disk';
5381         die "unable to change media type\n" if $media ne $oldmedia;
5382
5383         if (!drive_is_cdrom($old_drive)) {
5384
5385             if ($drive->{file} ne $old_drive->{file}) {
5386
5387                 die "skip\n" if !$hotplug;
5388
5389                 # unplug and register as unused
5390                 vm_deviceunplug($vmid, $conf, $opt);
5391                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5392
5393             } else {
5394                 # update existing disk
5395
5396                 # skip non hotpluggable value
5397                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5398                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5399                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5400                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5401                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5402                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5403                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5404                     die "skip\n";
5405                 }
5406
5407                 # apply throttle
5408                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5409                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5410                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5411                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5412                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5413                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5414                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5415                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5416                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5417                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5418                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5419                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5420                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5421                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5422                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5423                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5424                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5425                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5426
5427                     qemu_block_set_io_throttle(
5428                         $vmid,"drive-$opt",
5429                         ($drive->{mbps} || 0)*1024*1024,
5430                         ($drive->{mbps_rd} || 0)*1024*1024,
5431                         ($drive->{mbps_wr} || 0)*1024*1024,
5432                          $drive->{iops} || 0,
5433                          $drive->{iops_rd} || 0,
5434                          $drive->{iops_wr} || 0,
5435                         ($drive->{mbps_max} || 0)*1024*1024,
5436                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5437                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5438                          $drive->{iops_max} || 0,
5439                          $drive->{iops_rd_max} || 0,
5440                          $drive->{iops_wr_max} || 0,
5441                          $drive->{bps_max_length} || 1,
5442                          $drive->{bps_rd_max_length} || 1,
5443                          $drive->{bps_wr_max_length} || 1,
5444                          $drive->{iops_max_length} || 1,
5445                          $drive->{iops_rd_max_length} || 1,
5446                          $drive->{iops_wr_max_length} || 1,
5447                     );
5448
5449                 }
5450
5451                 return 1;
5452             }
5453
5454         } else { # cdrom
5455
5456             if ($drive->{file} eq 'none') {
5457                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5458                 if (drive_is_cloudinit($old_drive)) {
5459                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5460                 }
5461             } else {
5462                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5463
5464                 # force eject if locked
5465                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5466
5467                 if ($path) {
5468                     mon_cmd($vmid, "blockdev-change-medium",
5469                         id => "$opt", filename => "$path");
5470                 }
5471             }
5472
5473             return 1;
5474         }
5475     }
5476
5477     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5478     # hotplug new disks
5479     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5480     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5481 }
5482
5483 sub vmconfig_update_cloudinit_drive {
5484     my ($storecfg, $conf, $vmid) = @_;
5485
5486     my $cloudinit_ds = undef;
5487     my $cloudinit_drive = undef;
5488
5489     PVE::QemuConfig->foreach_volume($conf, sub {
5490         my ($ds, $drive) = @_;
5491         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5492             $cloudinit_ds = $ds;
5493             $cloudinit_drive = $drive;
5494         }
5495     });
5496
5497     return if !$cloudinit_drive;
5498
5499     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5500         PVE::QemuConfig->write_config($vmid, $conf);
5501     }
5502
5503     my $running = PVE::QemuServer::check_running($vmid);
5504
5505     if ($running) {
5506         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5507         if ($path) {
5508             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5509             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5510         }
5511     }
5512 }
5513
5514 # called in locked context by incoming migration
5515 sub vm_migrate_get_nbd_disks {
5516     my ($storecfg, $conf, $replicated_volumes) = @_;
5517
5518     my $local_volumes = {};
5519     PVE::QemuConfig->foreach_volume($conf, sub {
5520         my ($ds, $drive) = @_;
5521
5522         return if drive_is_cdrom($drive);
5523         return if $ds eq 'tpmstate0';
5524
5525         my $volid = $drive->{file};
5526
5527         return if !$volid;
5528
5529         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5530
5531         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5532         return if $scfg->{shared};
5533
5534         # replicated disks re-use existing state via bitmap
5535         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5536         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5537     });
5538     return $local_volumes;
5539 }
5540
5541 # called in locked context by incoming migration
5542 sub vm_migrate_alloc_nbd_disks {
5543     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5544
5545     my $nbd = {};
5546     foreach my $opt (sort keys %$source_volumes) {
5547         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5548
5549         if ($use_existing) {
5550             $nbd->{$opt}->{drivestr} = print_drive($drive);
5551             $nbd->{$opt}->{volid} = $volid;
5552             $nbd->{$opt}->{replicated} = 1;
5553             next;
5554         }
5555
5556         # storage mapping + volname = regular migration
5557         # storage mapping + format = remote migration
5558         # order of precedence, filtered by whether storage supports it:
5559         # 1. explicit requested format
5560         # 2. format of current volume
5561         # 3. default format of storage
5562         if (!$storagemap->{identity}) {
5563             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5564             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5565             if (!$format || !grep { $format eq $_ } @$validFormats) {
5566                 if ($volname) {
5567                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5568                     my $fileFormat = qemu_img_format($scfg, $volname);
5569                     $format = $fileFormat
5570                         if grep { $fileFormat eq $_ } @$validFormats;
5571                 }
5572                 $format //= $defFormat;
5573             }
5574         } else {
5575             # can't happen for remote migration, so $volname is always defined
5576             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5577             $format = qemu_img_format($scfg, $volname);
5578         }
5579
5580         my $size = $drive->{size} / 1024;
5581         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5582         my $newdrive = $drive;
5583         $newdrive->{format} = $format;
5584         $newdrive->{file} = $newvolid;
5585         my $drivestr = print_drive($newdrive);
5586         $nbd->{$opt}->{drivestr} = $drivestr;
5587         $nbd->{$opt}->{volid} = $newvolid;
5588     }
5589
5590     return $nbd;
5591 }
5592
5593 # see vm_start_nolock for parameters, additionally:
5594 # migrate_opts:
5595 #   storagemap = parsed storage map for allocating NBD disks
5596 sub vm_start {
5597     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5598
5599     return PVE::QemuConfig->lock_config($vmid, sub {
5600         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5601
5602         die "you can't start a vm if it's a template\n"
5603             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5604
5605         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5606         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5607
5608         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5609
5610         if ($has_backup_lock && $running) {
5611             # a backup is currently running, attempt to start the guest in the
5612             # existing QEMU instance
5613             return vm_resume($vmid);
5614         }
5615
5616         PVE::QemuConfig->check_lock($conf)
5617             if !($params->{skiplock} || $has_suspended_lock);
5618
5619         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5620
5621         die "VM $vmid already running\n" if $running;
5622
5623         if (my $storagemap = $migrate_opts->{storagemap}) {
5624             my $replicated = $migrate_opts->{replicated_volumes};
5625             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5626             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5627
5628             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5629                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5630             }
5631         }
5632
5633         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5634     });
5635 }
5636
5637
5638 # params:
5639 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5640 #   skiplock => 0/1, skip checking for config lock
5641 #   skiptemplate => 0/1, skip checking whether VM is template
5642 #   forcemachine => to force QEMU machine (rollback/migration)
5643 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5644 #   timeout => in seconds
5645 #   paused => start VM in paused state (backup)
5646 #   resume => resume from hibernation
5647 #   pbs-backing => {
5648 #      sata0 => {
5649 #         repository
5650 #         snapshot
5651 #         keyfile
5652 #         archive
5653 #      },
5654 #      virtio2 => ...
5655 #   }
5656 # migrate_opts:
5657 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5658 #   migratedfrom => source node
5659 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5660 #   network => CIDR of migration network
5661 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5662 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5663 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5664 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5665 #       contained in config
5666 sub vm_start_nolock {
5667     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5668
5669     my $statefile = $params->{statefile};
5670     my $resume = $params->{resume};
5671
5672     my $migratedfrom = $migrate_opts->{migratedfrom};
5673     my $migration_type = $migrate_opts->{type};
5674
5675     my $res = {};
5676
5677     # clean up leftover reboot request files
5678     eval { clear_reboot_request($vmid); };
5679     warn $@ if $@;
5680
5681     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5682         vmconfig_apply_pending($vmid, $conf, $storecfg);
5683         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5684     }
5685
5686     # don't regenerate the ISO if the VM is started as part of a live migration
5687     # this way we can reuse the old ISO with the correct config
5688     if (!$migratedfrom) {
5689         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5690             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5691             # $conf->{cloudinit}, so we could just not do this?
5692             # But we do it above, so for now let's be consistent.
5693             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5694         }
5695     }
5696
5697     # override offline migrated volumes, conf is out of date still
5698     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5699         for my $key (sort keys $offline_volumes->%*) {
5700             my $parsed = parse_drive($key, $conf->{$key});
5701             $parsed->{file} = $offline_volumes->{$key};
5702             $conf->{$key} = print_drive($parsed);
5703         }
5704     }
5705
5706     my $defaults = load_defaults();
5707
5708     # set environment variable useful inside network script
5709     # for remote migration the config is available on the target node!
5710     if (!$migrate_opts->{remote_node}) {
5711         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5712     }
5713
5714     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5715
5716     my $forcemachine = $params->{forcemachine};
5717     my $forcecpu = $params->{forcecpu};
5718     if ($resume) {
5719         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5720         $forcemachine = $conf->{runningmachine};
5721         $forcecpu = $conf->{runningcpu};
5722         print "Resuming suspended VM\n";
5723     }
5724
5725     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5726         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5727
5728     my $migration_ip;
5729     my $get_migration_ip = sub {
5730         my ($nodename) = @_;
5731
5732         return $migration_ip if defined($migration_ip);
5733
5734         my $cidr = $migrate_opts->{network};
5735
5736         if (!defined($cidr)) {
5737             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5738             $cidr = $dc_conf->{migration}->{network};
5739         }
5740
5741         if (defined($cidr)) {
5742             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5743
5744             die "could not get IP: no address configured on local " .
5745                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5746
5747             die "could not get IP: multiple addresses configured on local " .
5748                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5749
5750             $migration_ip = @$ips[0];
5751         }
5752
5753         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5754             if !defined($migration_ip);
5755
5756         return $migration_ip;
5757     };
5758
5759     if ($statefile) {
5760         if ($statefile eq 'tcp') {
5761             my $migrate = $res->{migrate} = { proto => 'tcp' };
5762             $migrate->{addr} = "localhost";
5763             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5764             my $nodename = nodename();
5765
5766             if (!defined($migration_type)) {
5767                 if (defined($datacenterconf->{migration}->{type})) {
5768                     $migration_type = $datacenterconf->{migration}->{type};
5769                 } else {
5770                     $migration_type = 'secure';
5771                 }
5772             }
5773
5774             if ($migration_type eq 'insecure') {
5775                 $migrate->{addr} = $get_migration_ip->($nodename);
5776                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5777             }
5778
5779             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5780             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5781             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5782             push @$cmd, '-incoming', $migrate->{uri};
5783             push @$cmd, '-S';
5784
5785         } elsif ($statefile eq 'unix') {
5786             # should be default for secure migrations as a ssh TCP forward
5787             # tunnel is not deterministic reliable ready and fails regurarly
5788             # to set up in time, so use UNIX socket forwards
5789             my $migrate = $res->{migrate} = { proto => 'unix' };
5790             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5791             unlink $migrate->{addr};
5792
5793             $migrate->{uri} = "unix:$migrate->{addr}";
5794             push @$cmd, '-incoming', $migrate->{uri};
5795             push @$cmd, '-S';
5796
5797         } elsif (-e $statefile) {
5798             push @$cmd, '-loadstate', $statefile;
5799         } else {
5800             my $statepath = PVE::Storage::path($storecfg, $statefile);
5801             push @$vollist, $statefile;
5802             push @$cmd, '-loadstate', $statepath;
5803         }
5804     } elsif ($params->{paused}) {
5805         push @$cmd, '-S';
5806     }
5807
5808     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5809
5810     my $pci_devices = {}; # host pci devices
5811     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5812         my $dev = $conf->{"hostpci$i"} or next;
5813         $pci_devices->{$i} = parse_hostpci($dev);
5814     }
5815
5816     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5817     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5818
5819     # map to a flat list of pci ids
5820     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5821
5822     # reserve all PCI IDs before actually doing anything with them
5823     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5824
5825     eval {
5826         my $uuid;
5827         for my $id (sort keys %$pci_devices) {
5828             my $d = $pci_devices->{$id};
5829             for my $dev ($d->{pciid}->@*) {
5830                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5831
5832                 # nvidia grid needs the uuid of the mdev as qemu parameter
5833                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5834                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5835                 }
5836             }
5837         }
5838         push @$cmd, '-uuid', $uuid if defined($uuid);
5839     };
5840     if (my $err = $@) {
5841         eval { cleanup_pci_devices($vmid, $conf) };
5842         warn $@ if $@;
5843         die $err;
5844     }
5845
5846     PVE::Storage::activate_volumes($storecfg, $vollist);
5847
5848     eval {
5849         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5850     };
5851     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5852     # timeout should be more than enough here...
5853     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5854
5855     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5856
5857     my %run_params = (
5858         timeout => $statefile ? undef : $start_timeout,
5859         umask => 0077,
5860         noerr => 1,
5861     );
5862
5863     # when migrating, prefix QEMU output so other side can pick up any
5864     # errors that might occur and show the user
5865     if ($migratedfrom) {
5866         $run_params{quiet} = 1;
5867         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5868     }
5869
5870     my %systemd_properties = (
5871         Slice => 'qemu.slice',
5872         KillMode => 'process',
5873         SendSIGKILL => 0,
5874         TimeoutStopUSec => ULONG_MAX, # infinity
5875     );
5876
5877     if (PVE::CGroup::cgroup_mode() == 2) {
5878         $systemd_properties{CPUWeight} = $cpuunits;
5879     } else {
5880         $systemd_properties{CPUShares} = $cpuunits;
5881     }
5882
5883     if (my $cpulimit = $conf->{cpulimit}) {
5884         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5885     }
5886     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5887
5888     my $run_qemu = sub {
5889         PVE::Tools::run_fork sub {
5890             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5891
5892             my $tpmpid;
5893             if (my $tpm = $conf->{tpmstate0}) {
5894                 # start the TPM emulator so QEMU can connect on start
5895                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5896             }
5897
5898             my $exitcode = run_command($cmd, %run_params);
5899             if ($exitcode) {
5900                 if ($tpmpid) {
5901                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5902                     kill 'TERM', $tpmpid;
5903                 }
5904                 die "QEMU exited with code $exitcode\n";
5905             }
5906         };
5907     };
5908
5909     if ($conf->{hugepages}) {
5910
5911         my $code = sub {
5912             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5913             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5914
5915             PVE::QemuServer::Memory::hugepages_mount();
5916             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5917
5918             eval { $run_qemu->() };
5919             if (my $err = $@) {
5920                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5921                     if !$conf->{keephugepages};
5922                 die $err;
5923             }
5924
5925             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5926                 if !$conf->{keephugepages};
5927         };
5928         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5929
5930     } else {
5931         eval { $run_qemu->() };
5932     }
5933
5934     if (my $err = $@) {
5935         # deactivate volumes if start fails
5936         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5937         warn $@ if $@;
5938         eval { cleanup_pci_devices($vmid, $conf) };
5939         warn $@ if $@;
5940
5941         die "start failed: $err";
5942     }
5943
5944     # re-reserve all PCI IDs now that we can know the actual VM PID
5945     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5946     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5947     warn $@ if $@;
5948
5949     if (defined($res->{migrate})) {
5950         print "migration listens on $res->{migrate}->{uri}\n";
5951     } elsif ($statefile) {
5952         eval { mon_cmd($vmid, "cont"); };
5953         warn $@ if $@;
5954     }
5955
5956     #start nbd server for storage migration
5957     if (my $nbd = $migrate_opts->{nbd}) {
5958         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5959
5960         my $migrate_storage_uri;
5961         # nbd_protocol_version > 0 for unix socket support
5962         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5963             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5964             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5965             $migrate_storage_uri = "nbd:unix:$socket_path";
5966             $res->{migrate}->{unix_sockets} = [$socket_path];
5967         } else {
5968             my $nodename = nodename();
5969             my $localip = $get_migration_ip->($nodename);
5970             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5971             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5972
5973             mon_cmd($vmid, "nbd-server-start", addr => {
5974                 type => 'inet',
5975                 data => {
5976                     host => "${localip}",
5977                     port => "${storage_migrate_port}",
5978                 },
5979             });
5980             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5981             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5982         }
5983
5984         my $block_info = mon_cmd($vmid, "query-block");
5985         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5986
5987         foreach my $opt (sort keys %$nbd) {
5988             my $drivestr = $nbd->{$opt}->{drivestr};
5989             my $volid = $nbd->{$opt}->{volid};
5990
5991             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5992
5993             mon_cmd(
5994                 $vmid,
5995                 "block-export-add",
5996                 id => "drive-$opt",
5997                 'node-name' => $block_node,
5998                 writable => JSON::true,
5999                 type => "nbd",
6000                 name => "drive-$opt", # NBD export name
6001             );
6002
6003             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6004             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6005             print "re-using replicated volume: $opt - $volid\n"
6006                 if $nbd->{$opt}->{replicated};
6007
6008             $res->{drives}->{$opt} = $nbd->{$opt};
6009             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6010         }
6011     }
6012
6013     if ($migratedfrom) {
6014         eval {
6015             set_migration_caps($vmid);
6016         };
6017         warn $@ if $@;
6018
6019         if ($spice_port) {
6020             print "spice listens on port $spice_port\n";
6021             $res->{spice_port} = $spice_port;
6022             if ($migrate_opts->{spice_ticket}) {
6023                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6024                     $migrate_opts->{spice_ticket});
6025                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6026             }
6027         }
6028
6029     } else {
6030         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6031             if !$statefile && $conf->{balloon};
6032
6033         foreach my $opt (keys %$conf) {
6034             next if $opt !~  m/^net\d+$/;
6035             my $nicconf = parse_net($conf->{$opt});
6036             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6037         }
6038         add_nets_bridge_fdb($conf, $vmid);
6039     }
6040
6041    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6042         mon_cmd(
6043             $vmid,
6044             'qom-set',
6045             path => "machine/peripheral/balloon0",
6046             property => "guest-stats-polling-interval",
6047             value => 2
6048         );
6049     }
6050
6051     if ($resume) {
6052         print "Resumed VM, removing state\n";
6053         if (my $vmstate = $conf->{vmstate}) {
6054             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6055             PVE::Storage::vdisk_free($storecfg, $vmstate);
6056         }
6057         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6058         PVE::QemuConfig->write_config($vmid, $conf);
6059     }
6060
6061     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6062
6063     return $res;
6064 }
6065
6066 sub vm_commandline {
6067     my ($storecfg, $vmid, $snapname) = @_;
6068
6069     my $conf = PVE::QemuConfig->load_config($vmid);
6070
6071     my ($forcemachine, $forcecpu);
6072     if ($snapname) {
6073         my $snapshot = $conf->{snapshots}->{$snapname};
6074         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6075
6076         # check for machine or CPU overrides in snapshot
6077         $forcemachine = $snapshot->{runningmachine};
6078         $forcecpu = $snapshot->{runningcpu};
6079
6080         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6081
6082         $conf = $snapshot;
6083     }
6084
6085     my $defaults = load_defaults();
6086
6087     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6088
6089     return PVE::Tools::cmd2string($cmd);
6090 }
6091
6092 sub vm_reset {
6093     my ($vmid, $skiplock) = @_;
6094
6095     PVE::QemuConfig->lock_config($vmid, sub {
6096
6097         my $conf = PVE::QemuConfig->load_config($vmid);
6098
6099         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6100
6101         mon_cmd($vmid, "system_reset");
6102     });
6103 }
6104
6105 sub get_vm_volumes {
6106     my ($conf) = @_;
6107
6108     my $vollist = [];
6109     foreach_volid($conf, sub {
6110         my ($volid, $attr) = @_;
6111
6112         return if $volid =~ m|^/|;
6113
6114         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6115         return if !$sid;
6116
6117         push @$vollist, $volid;
6118     });
6119
6120     return $vollist;
6121 }
6122
6123 sub cleanup_pci_devices {
6124     my ($vmid, $conf) = @_;
6125
6126     foreach my $key (keys %$conf) {
6127         next if $key !~ m/^hostpci(\d+)$/;
6128         my $hostpciindex = $1;
6129         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6130         my $d = parse_hostpci($conf->{$key});
6131         if ($d->{mdev}) {
6132             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6133             # don't want to break ABI just for this two liner
6134             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6135             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6136         }
6137     }
6138     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6139 }
6140
6141 sub vm_stop_cleanup {
6142     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6143
6144     eval {
6145
6146         if (!$keepActive) {
6147             my $vollist = get_vm_volumes($conf);
6148             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6149
6150             if (my $tpmdrive = $conf->{tpmstate0}) {
6151                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6152                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6153                 if ($storeid) {
6154                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6155                 }
6156             }
6157         }
6158
6159         foreach my $ext (qw(mon qmp pid vnc qga)) {
6160             unlink "/var/run/qemu-server/${vmid}.$ext";
6161         }
6162
6163         if ($conf->{ivshmem}) {
6164             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6165             # just delete it for now, VMs which have this already open do not
6166             # are affected, but new VMs will get a separated one. If this
6167             # becomes an issue we either add some sort of ref-counting or just
6168             # add a "don't delete on stop" flag to the ivshmem format.
6169             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6170         }
6171
6172         cleanup_pci_devices($vmid, $conf);
6173
6174         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6175     };
6176     warn $@ if $@; # avoid errors - just warn
6177 }
6178
6179 # call only in locked context
6180 sub _do_vm_stop {
6181     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6182
6183     my $pid = check_running($vmid, $nocheck);
6184     return if !$pid;
6185
6186     my $conf;
6187     if (!$nocheck) {
6188         $conf = PVE::QemuConfig->load_config($vmid);
6189         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6190         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6191             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6192             $timeout = $opts->{down} if $opts->{down};
6193         }
6194         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6195     }
6196
6197     eval {
6198         if ($shutdown) {
6199             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6200                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6201             } else {
6202                 mon_cmd($vmid, "system_powerdown");
6203             }
6204         } else {
6205             mon_cmd($vmid, "quit");
6206         }
6207     };
6208     my $err = $@;
6209
6210     if (!$err) {
6211         $timeout = 60 if !defined($timeout);
6212
6213         my $count = 0;
6214         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6215             $count++;
6216             sleep 1;
6217         }
6218
6219         if ($count >= $timeout) {
6220             if ($force) {
6221                 warn "VM still running - terminating now with SIGTERM\n";
6222                 kill 15, $pid;
6223             } else {
6224                 die "VM quit/powerdown failed - got timeout\n";
6225             }
6226         } else {
6227             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6228             return;
6229         }
6230     } else {
6231        if (!check_running($vmid, $nocheck)) {
6232             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6233             return;
6234        }
6235        if ($force) {
6236             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6237             kill 15, $pid;
6238         } else {
6239             die "VM quit/powerdown failed\n";
6240         }
6241     }
6242
6243     # wait again
6244     $timeout = 10;
6245
6246     my $count = 0;
6247     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6248         $count++;
6249         sleep 1;
6250     }
6251
6252     if ($count >= $timeout) {
6253         warn "VM still running - terminating now with SIGKILL\n";
6254         kill 9, $pid;
6255         sleep 1;
6256     }
6257
6258     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6259 }
6260
6261 # Note: use $nocheck to skip tests if VM configuration file exists.
6262 # We need that when migration VMs to other nodes (files already moved)
6263 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6264 sub vm_stop {
6265     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6266
6267     $force = 1 if !defined($force) && !$shutdown;
6268
6269     if ($migratedfrom){
6270         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6271         kill 15, $pid if $pid;
6272         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6273         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6274         return;
6275     }
6276
6277     PVE::QemuConfig->lock_config($vmid, sub {
6278         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6279    });
6280 }
6281
6282 sub vm_reboot {
6283     my ($vmid, $timeout) = @_;
6284
6285     PVE::QemuConfig->lock_config($vmid, sub {
6286         eval {
6287
6288             # only reboot if running, as qmeventd starts it again on a stop event
6289             return if !check_running($vmid);
6290
6291             create_reboot_request($vmid);
6292
6293             my $storecfg = PVE::Storage::config();
6294             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6295
6296         };
6297         if (my $err = $@) {
6298             # avoid that the next normal shutdown will be confused for a reboot
6299             clear_reboot_request($vmid);
6300             die $err;
6301         }
6302    });
6303 }
6304
6305 # note: if using the statestorage parameter, the caller has to check privileges
6306 sub vm_suspend {
6307     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6308
6309     my $conf;
6310     my $path;
6311     my $storecfg;
6312     my $vmstate;
6313
6314     PVE::QemuConfig->lock_config($vmid, sub {
6315
6316         $conf = PVE::QemuConfig->load_config($vmid);
6317
6318         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6319         PVE::QemuConfig->check_lock($conf)
6320             if !($skiplock || $is_backing_up);
6321
6322         die "cannot suspend to disk during backup\n"
6323             if $is_backing_up && $includestate;
6324
6325         if ($includestate) {
6326             $conf->{lock} = 'suspending';
6327             my $date = strftime("%Y-%m-%d", localtime(time()));
6328             $storecfg = PVE::Storage::config();
6329             if (!$statestorage) {
6330                 $statestorage = find_vmstate_storage($conf, $storecfg);
6331                 # check permissions for the storage
6332                 my $rpcenv = PVE::RPCEnvironment::get();
6333                 if ($rpcenv->{type} ne 'cli') {
6334                     my $authuser = $rpcenv->get_user();
6335                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6336                 }
6337             }
6338
6339
6340             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6341                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6342             $path = PVE::Storage::path($storecfg, $vmstate);
6343             PVE::QemuConfig->write_config($vmid, $conf);
6344         } else {
6345             mon_cmd($vmid, "stop");
6346         }
6347     });
6348
6349     if ($includestate) {
6350         # save vm state
6351         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6352
6353         eval {
6354             set_migration_caps($vmid, 1);
6355             mon_cmd($vmid, "savevm-start", statefile => $path);
6356             for(;;) {
6357                 my $state = mon_cmd($vmid, "query-savevm");
6358                 if (!$state->{status}) {
6359                     die "savevm not active\n";
6360                 } elsif ($state->{status} eq 'active') {
6361                     sleep(1);
6362                     next;
6363                 } elsif ($state->{status} eq 'completed') {
6364                     print "State saved, quitting\n";
6365                     last;
6366                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6367                     die "query-savevm failed with error '$state->{error}'\n"
6368                 } else {
6369                     die "query-savevm returned status '$state->{status}'\n";
6370                 }
6371             }
6372         };
6373         my $err = $@;
6374
6375         PVE::QemuConfig->lock_config($vmid, sub {
6376             $conf = PVE::QemuConfig->load_config($vmid);
6377             if ($err) {
6378                 # cleanup, but leave suspending lock, to indicate something went wrong
6379                 eval {
6380                     mon_cmd($vmid, "savevm-end");
6381                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6382                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6383                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6384                     PVE::QemuConfig->write_config($vmid, $conf);
6385                 };
6386                 warn $@ if $@;
6387                 die $err;
6388             }
6389
6390             die "lock changed unexpectedly\n"
6391                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6392
6393             mon_cmd($vmid, "quit");
6394             $conf->{lock} = 'suspended';
6395             PVE::QemuConfig->write_config($vmid, $conf);
6396         });
6397     }
6398 }
6399
6400 # $nocheck is set when called as part of a migration - in this context the
6401 # location of the config file (source or target node) is not deterministic,
6402 # since migration cannot wait for pmxcfs to process the rename
6403 sub vm_resume {
6404     my ($vmid, $skiplock, $nocheck) = @_;
6405
6406     PVE::QemuConfig->lock_config($vmid, sub {
6407         my $res = mon_cmd($vmid, 'query-status');
6408         my $resume_cmd = 'cont';
6409         my $reset = 0;
6410         my $conf;
6411         if ($nocheck) {
6412             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6413             if ($@) {
6414                 my $vmlist = PVE::Cluster::get_vmlist();
6415                 if (exists($vmlist->{ids}->{$vmid})) {
6416                     my $node = $vmlist->{ids}->{$vmid}->{node};
6417                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6418                 }
6419                 if (!$conf) {
6420                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6421                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6422                 }
6423             }
6424         } else {
6425             $conf = PVE::QemuConfig->load_config($vmid);
6426         }
6427
6428         if ($res->{status}) {
6429             return if $res->{status} eq 'running'; # job done, go home
6430             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6431             $reset = 1 if $res->{status} eq 'shutdown';
6432         }
6433
6434         if (!$nocheck) {
6435             PVE::QemuConfig->check_lock($conf)
6436                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6437         }
6438
6439         if ($reset) {
6440             # required if a VM shuts down during a backup and we get a resume
6441             # request before the backup finishes for example
6442             mon_cmd($vmid, "system_reset");
6443         }
6444
6445         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6446
6447         mon_cmd($vmid, $resume_cmd);
6448     });
6449 }
6450
6451 sub vm_sendkey {
6452     my ($vmid, $skiplock, $key) = @_;
6453
6454     PVE::QemuConfig->lock_config($vmid, sub {
6455
6456         my $conf = PVE::QemuConfig->load_config($vmid);
6457
6458         # there is no qmp command, so we use the human monitor command
6459         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6460         die $res if $res ne '';
6461     });
6462 }
6463
6464 # vzdump restore implementaion
6465
6466 sub tar_archive_read_firstfile {
6467     my $archive = shift;
6468
6469     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6470
6471     # try to detect archive type first
6472     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6473         die "unable to open file '$archive'\n";
6474     my $firstfile = <$fh>;
6475     kill 15, $pid;
6476     close $fh;
6477
6478     die "ERROR: archive contaions no data\n" if !$firstfile;
6479     chomp $firstfile;
6480
6481     return $firstfile;
6482 }
6483
6484 sub tar_restore_cleanup {
6485     my ($storecfg, $statfile) = @_;
6486
6487     print STDERR "starting cleanup\n";
6488
6489     if (my $fd = IO::File->new($statfile, "r")) {
6490         while (defined(my $line = <$fd>)) {
6491             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6492                 my $volid = $2;
6493                 eval {
6494                     if ($volid =~ m|^/|) {
6495                         unlink $volid || die 'unlink failed\n';
6496                     } else {
6497                         PVE::Storage::vdisk_free($storecfg, $volid);
6498                     }
6499                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6500                 };
6501                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6502             } else {
6503                 print STDERR "unable to parse line in statfile - $line";
6504             }
6505         }
6506         $fd->close();
6507     }
6508 }
6509
6510 sub restore_file_archive {
6511     my ($archive, $vmid, $user, $opts) = @_;
6512
6513     return restore_vma_archive($archive, $vmid, $user, $opts)
6514         if $archive eq '-';
6515
6516     my $info = PVE::Storage::archive_info($archive);
6517     my $format = $opts->{format} // $info->{format};
6518     my $comp = $info->{compression};
6519
6520     # try to detect archive format
6521     if ($format eq 'tar') {
6522         return restore_tar_archive($archive, $vmid, $user, $opts);
6523     } else {
6524         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6525     }
6526 }
6527
6528 # hepler to remove disks that will not be used after restore
6529 my $restore_cleanup_oldconf = sub {
6530     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6531
6532     my $kept_disks = {};
6533
6534     PVE::QemuConfig->foreach_volume($oldconf, sub {
6535         my ($ds, $drive) = @_;
6536
6537         return if drive_is_cdrom($drive, 1);
6538
6539         my $volid = $drive->{file};
6540         return if !$volid || $volid =~ m|^/|;
6541
6542         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6543         return if !$path || !$owner || ($owner != $vmid);
6544
6545         # Note: only delete disk we want to restore
6546         # other volumes will become unused
6547         if ($virtdev_hash->{$ds}) {
6548             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6549             if (my $err = $@) {
6550                 warn $err;
6551             }
6552         } else {
6553             $kept_disks->{$volid} = 1;
6554         }
6555     });
6556
6557     # after the restore we have no snapshots anymore
6558     for my $snapname (keys $oldconf->{snapshots}->%*) {
6559         my $snap = $oldconf->{snapshots}->{$snapname};
6560         if ($snap->{vmstate}) {
6561             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6562             if (my $err = $@) {
6563                 warn $err;
6564             }
6565         }
6566
6567         for my $volid (keys $kept_disks->%*) {
6568             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6569             warn $@ if $@;
6570         }
6571     }
6572 };
6573
6574 # Helper to parse vzdump backup device hints
6575 #
6576 # $rpcenv: Environment, used to ckeck storage permissions
6577 # $user: User ID, to check storage permissions
6578 # $storecfg: Storage configuration
6579 # $fh: the file handle for reading the configuration
6580 # $devinfo: should contain device sizes for all backu-up'ed devices
6581 # $options: backup options (pool, default storage)
6582 #
6583 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6584 my $parse_backup_hints = sub {
6585     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6586
6587     my $check_storage = sub { # assert if an image can be allocate
6588         my ($storeid, $scfg) = @_;
6589         die "Content type 'images' is not available on storage '$storeid'\n"
6590             if !$scfg->{content}->{images};
6591         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6592             if $user ne 'root@pam';
6593     };
6594
6595     my $virtdev_hash = {};
6596     while (defined(my $line = <$fh>)) {
6597         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6598             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6599             die "archive does not contain data for drive '$virtdev'\n"
6600                 if !$devinfo->{$devname};
6601
6602             if (defined($options->{storage})) {
6603                 $storeid = $options->{storage} || 'local';
6604             } elsif (!$storeid) {
6605                 $storeid = 'local';
6606             }
6607             $format = 'raw' if !$format;
6608             $devinfo->{$devname}->{devname} = $devname;
6609             $devinfo->{$devname}->{virtdev} = $virtdev;
6610             $devinfo->{$devname}->{format} = $format;
6611             $devinfo->{$devname}->{storeid} = $storeid;
6612
6613             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6614             $check_storage->($storeid, $scfg); # permission and content type check
6615
6616             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6617         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6618             my $virtdev = $1;
6619             my $drive = parse_drive($virtdev, $2);
6620
6621             if (drive_is_cloudinit($drive)) {
6622                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6623                 $storeid = $options->{storage} if defined ($options->{storage});
6624                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6625                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6626
6627                 $check_storage->($storeid, $scfg); # permission and content type check
6628
6629                 $virtdev_hash->{$virtdev} = {
6630                     format => $format,
6631                     storeid => $storeid,
6632                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6633                     is_cloudinit => 1,
6634                 };
6635             }
6636         }
6637     }
6638
6639     return $virtdev_hash;
6640 };
6641
6642 # Helper to allocate and activate all volumes required for a restore
6643 #
6644 # $storecfg: Storage configuration
6645 # $virtdev_hash: as returned by parse_backup_hints()
6646 #
6647 # Returns: { $virtdev => $volid }
6648 my $restore_allocate_devices = sub {
6649     my ($storecfg, $virtdev_hash, $vmid) = @_;
6650
6651     my $map = {};
6652     foreach my $virtdev (sort keys %$virtdev_hash) {
6653         my $d = $virtdev_hash->{$virtdev};
6654         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6655         my $storeid = $d->{storeid};
6656         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6657
6658         # test if requested format is supported
6659         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6660         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6661         $d->{format} = $defFormat if !$supported;
6662
6663         my $name;
6664         if ($d->{is_cloudinit}) {
6665             $name = "vm-$vmid-cloudinit";
6666             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6667             if ($scfg->{path}) {
6668                 $name .= ".$d->{format}";
6669             }
6670         }
6671
6672         my $volid = PVE::Storage::vdisk_alloc(
6673             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6674
6675         print STDERR "new volume ID is '$volid'\n";
6676         $d->{volid} = $volid;
6677
6678         PVE::Storage::activate_volumes($storecfg, [$volid]);
6679
6680         $map->{$virtdev} = $volid;
6681     }
6682
6683     return $map;
6684 };
6685
6686 sub restore_update_config_line {
6687     my ($cookie, $map, $line, $unique) = @_;
6688
6689     return '' if $line =~ m/^\#qmdump\#/;
6690     return '' if $line =~ m/^\#vzdump\#/;
6691     return '' if $line =~ m/^lock:/;
6692     return '' if $line =~ m/^unused\d+:/;
6693     return '' if $line =~ m/^parent:/;
6694
6695     my $res = '';
6696
6697     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6698     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6699         # try to convert old 1.X settings
6700         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6701         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6702             my ($model, $macaddr) = split(/\=/, $devconfig);
6703             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6704             my $net = {
6705                 model => $model,
6706                 bridge => "vmbr$ind",
6707                 macaddr => $macaddr,
6708             };
6709             my $netstr = print_net($net);
6710
6711             $res .= "net$cookie->{netcount}: $netstr\n";
6712             $cookie->{netcount}++;
6713         }
6714     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6715         my ($id, $netstr) = ($1, $2);
6716         my $net = parse_net($netstr);
6717         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6718         $netstr = print_net($net);
6719         $res .= "$id: $netstr\n";
6720     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6721         my $virtdev = $1;
6722         my $value = $3;
6723         my $di = parse_drive($virtdev, $value);
6724         if (defined($di->{backup}) && !$di->{backup}) {
6725             $res .= "#$line";
6726         } elsif ($map->{$virtdev}) {
6727             delete $di->{format}; # format can change on restore
6728             $di->{file} = $map->{$virtdev};
6729             $value = print_drive($di);
6730             $res .= "$virtdev: $value\n";
6731         } else {
6732             $res .= $line;
6733         }
6734     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6735         my $vmgenid = $1;
6736         if ($vmgenid ne '0') {
6737             # always generate a new vmgenid if there was a valid one setup
6738             $vmgenid = generate_uuid();
6739         }
6740         $res .= "vmgenid: $vmgenid\n";
6741     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6742         my ($uuid, $uuid_str);
6743         UUID::generate($uuid);
6744         UUID::unparse($uuid, $uuid_str);
6745         my $smbios1 = parse_smbios1($2);
6746         $smbios1->{uuid} = $uuid_str;
6747         $res .= $1.print_smbios1($smbios1)."\n";
6748     } else {
6749         $res .= $line;
6750     }
6751
6752     return $res;
6753 }
6754
6755 my $restore_deactivate_volumes = sub {
6756     my ($storecfg, $virtdev_hash) = @_;
6757
6758     my $vollist = [];
6759     for my $dev (values $virtdev_hash->%*) {
6760         push $vollist->@*, $dev->{volid} if $dev->{volid};
6761     }
6762
6763     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6764     print STDERR $@ if $@;
6765 };
6766
6767 my $restore_destroy_volumes = sub {
6768     my ($storecfg, $virtdev_hash) = @_;
6769
6770     for my $dev (values $virtdev_hash->%*) {
6771         my $volid = $dev->{volid} or next;
6772         eval {
6773             PVE::Storage::vdisk_free($storecfg, $volid);
6774             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6775         };
6776         print STDERR "unable to cleanup '$volid' - $@" if $@;
6777     }
6778 };
6779
6780 my $restore_merge_config = sub {
6781     my ($filename, $backup_conf_raw, $override_conf) = @_;
6782
6783     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6784     for my $key (keys $override_conf->%*) {
6785         $backup_conf->{$key} = $override_conf->{$key};
6786     }
6787
6788     return $backup_conf;
6789 };
6790
6791 sub scan_volids {
6792     my ($cfg, $vmid) = @_;
6793
6794     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6795
6796     my $volid_hash = {};
6797     foreach my $storeid (keys %$info) {
6798         foreach my $item (@{$info->{$storeid}}) {
6799             next if !($item->{volid} && $item->{size});
6800             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6801             $volid_hash->{$item->{volid}} = $item;
6802         }
6803     }
6804
6805     return $volid_hash;
6806 }
6807
6808 sub update_disk_config {
6809     my ($vmid, $conf, $volid_hash) = @_;
6810
6811     my $changes;
6812     my $prefix = "VM $vmid";
6813
6814     # used and unused disks
6815     my $referenced = {};
6816
6817     # Note: it is allowed to define multiple storages with same path (alias), so
6818     # we need to check both 'volid' and real 'path' (two different volid can point
6819     # to the same path).
6820
6821     my $referencedpath = {};
6822
6823     # update size info
6824     PVE::QemuConfig->foreach_volume($conf, sub {
6825         my ($opt, $drive) = @_;
6826
6827         my $volid = $drive->{file};
6828         return if !$volid;
6829         my $volume = $volid_hash->{$volid};
6830
6831         # mark volid as "in-use" for next step
6832         $referenced->{$volid} = 1;
6833         if ($volume && (my $path = $volume->{path})) {
6834             $referencedpath->{$path} = 1;
6835         }
6836
6837         return if drive_is_cdrom($drive);
6838         return if !$volume;
6839
6840         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6841         if (defined($updated)) {
6842             $changes = 1;
6843             $conf->{$opt} = print_drive($updated);
6844             print "$prefix ($opt): $msg\n";
6845         }
6846     });
6847
6848     # remove 'unusedX' entry if volume is used
6849     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6850         my ($opt, $drive) = @_;
6851
6852         my $volid = $drive->{file};
6853         return if !$volid;
6854
6855         my $path;
6856         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6857         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6858             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6859             $changes = 1;
6860             delete $conf->{$opt};
6861         }
6862
6863         $referenced->{$volid} = 1;
6864         $referencedpath->{$path} = 1 if $path;
6865     });
6866
6867     foreach my $volid (sort keys %$volid_hash) {
6868         next if $volid =~ m/vm-$vmid-state-/;
6869         next if $referenced->{$volid};
6870         my $path = $volid_hash->{$volid}->{path};
6871         next if !$path; # just to be sure
6872         next if $referencedpath->{$path};
6873         $changes = 1;
6874         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6875         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6876         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6877     }
6878
6879     return $changes;
6880 }
6881
6882 sub rescan {
6883     my ($vmid, $nolock, $dryrun) = @_;
6884
6885     my $cfg = PVE::Storage::config();
6886
6887     print "rescan volumes...\n";
6888     my $volid_hash = scan_volids($cfg, $vmid);
6889
6890     my $updatefn =  sub {
6891         my ($vmid) = @_;
6892
6893         my $conf = PVE::QemuConfig->load_config($vmid);
6894
6895         PVE::QemuConfig->check_lock($conf);
6896
6897         my $vm_volids = {};
6898         foreach my $volid (keys %$volid_hash) {
6899             my $info = $volid_hash->{$volid};
6900             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6901         }
6902
6903         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6904
6905         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6906     };
6907
6908     if (defined($vmid)) {
6909         if ($nolock) {
6910             &$updatefn($vmid);
6911         } else {
6912             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6913         }
6914     } else {
6915         my $vmlist = config_list();
6916         foreach my $vmid (keys %$vmlist) {
6917             if ($nolock) {
6918                 &$updatefn($vmid);
6919             } else {
6920                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6921             }
6922         }
6923     }
6924 }
6925
6926 sub restore_proxmox_backup_archive {
6927     my ($archive, $vmid, $user, $options) = @_;
6928
6929     my $storecfg = PVE::Storage::config();
6930
6931     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6932     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6933
6934     my $fingerprint = $scfg->{fingerprint};
6935     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6936
6937     my $repo = PVE::PBSClient::get_repository($scfg);
6938     my $namespace = $scfg->{namespace};
6939
6940     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6941     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6942     local $ENV{PBS_PASSWORD} = $password;
6943     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6944
6945     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6946         PVE::Storage::parse_volname($storecfg, $archive);
6947
6948     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6949
6950     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6951
6952     my $tmpdir = "/var/tmp/vzdumptmp$$";
6953     rmtree $tmpdir;
6954     mkpath $tmpdir;
6955
6956     my $conffile = PVE::QemuConfig->config_file($vmid);
6957      # disable interrupts (always do cleanups)
6958     local $SIG{INT} =
6959         local $SIG{TERM} =
6960         local $SIG{QUIT} =
6961         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6962
6963     # Note: $oldconf is undef if VM does not exists
6964     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6965     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6966     my $new_conf_raw = '';
6967
6968     my $rpcenv = PVE::RPCEnvironment::get();
6969     my $devinfo = {}; # info about drives included in backup
6970     my $virtdev_hash = {}; # info about allocated drives
6971
6972     eval {
6973         # enable interrupts
6974         local $SIG{INT} =
6975             local $SIG{TERM} =
6976             local $SIG{QUIT} =
6977             local $SIG{HUP} =
6978             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6979
6980         my $cfgfn = "$tmpdir/qemu-server.conf";
6981         my $firewall_config_fn = "$tmpdir/fw.conf";
6982         my $index_fn = "$tmpdir/index.json";
6983
6984         my $cmd = "restore";
6985
6986         my $param = [$pbs_backup_name, "index.json", $index_fn];
6987         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6988         my $index = PVE::Tools::file_get_contents($index_fn);
6989         $index = decode_json($index);
6990
6991         foreach my $info (@{$index->{files}}) {
6992             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6993                 my $devname = $1;
6994                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6995                     $devinfo->{$devname}->{size} = $1;
6996                 } else {
6997                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6998                 }
6999             }
7000         }
7001
7002         my $is_qemu_server_backup = scalar(
7003             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7004         );
7005         if (!$is_qemu_server_backup) {
7006             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7007         }
7008         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7009
7010         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7011         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7012
7013         if ($has_firewall_config) {
7014             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7015             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7016
7017             my $pve_firewall_dir = '/etc/pve/firewall';
7018             mkdir $pve_firewall_dir; # make sure the dir exists
7019             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7020         }
7021
7022         my $fh = IO::File->new($cfgfn, "r") ||
7023             die "unable to read qemu-server.conf - $!\n";
7024
7025         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7026
7027         # fixme: rate limit?
7028
7029         # create empty/temp config
7030         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7031
7032         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7033
7034         # allocate volumes
7035         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7036
7037         foreach my $virtdev (sort keys %$virtdev_hash) {
7038             my $d = $virtdev_hash->{$virtdev};
7039             next if $d->{is_cloudinit}; # no need to restore cloudinit
7040
7041             # this fails if storage is unavailable
7042             my $volid = $d->{volid};
7043             my $path = PVE::Storage::path($storecfg, $volid);
7044
7045             # for live-restore we only want to preload the efidisk and TPM state
7046             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7047
7048             my @ns_arg;
7049             if (defined(my $ns = $scfg->{namespace})) {
7050                 @ns_arg = ('--ns', $ns);
7051             }
7052
7053             my $pbs_restore_cmd = [
7054                 '/usr/bin/pbs-restore',
7055                 '--repository', $repo,
7056                 @ns_arg,
7057                 $pbs_backup_name,
7058                 "$d->{devname}.img.fidx",
7059                 $path,
7060                 '--verbose',
7061                 ];
7062
7063             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7064             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7065
7066             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7067                 push @$pbs_restore_cmd, '--skip-zero';
7068             }
7069
7070             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7071             print "restore proxmox backup image: $dbg_cmdstring\n";
7072             run_command($pbs_restore_cmd);
7073         }
7074
7075         $fh->seek(0, 0) || die "seek failed - $!\n";
7076
7077         my $cookie = { netcount => 0 };
7078         while (defined(my $line = <$fh>)) {
7079             $new_conf_raw .= restore_update_config_line(
7080                 $cookie,
7081                 $map,
7082                 $line,
7083                 $options->{unique},
7084             );
7085         }
7086
7087         $fh->close();
7088     };
7089     my $err = $@;
7090
7091     if ($err || !$options->{live}) {
7092         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7093     }
7094
7095     rmtree $tmpdir;
7096
7097     if ($err) {
7098         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7099         die $err;
7100     }
7101
7102     if ($options->{live}) {
7103         # keep lock during live-restore
7104         $new_conf_raw .= "\nlock: create";
7105     }
7106
7107     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7108     PVE::QemuConfig->write_config($vmid, $new_conf);
7109
7110     eval { rescan($vmid, 1); };
7111     warn $@ if $@;
7112
7113     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7114
7115     if ($options->{live}) {
7116         # enable interrupts
7117         local $SIG{INT} =
7118             local $SIG{TERM} =
7119             local $SIG{QUIT} =
7120             local $SIG{HUP} =
7121             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7122
7123         my $conf = PVE::QemuConfig->load_config($vmid);
7124         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7125
7126         # these special drives are already restored before start
7127         delete $devinfo->{'drive-efidisk0'};
7128         delete $devinfo->{'drive-tpmstate0-backup'};
7129
7130         my $pbs_opts = {
7131             repo => $repo,
7132             keyfile => $keyfile,
7133             snapshot => $pbs_backup_name,
7134             namespace => $namespace,
7135         };
7136         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7137
7138         PVE::QemuConfig->remove_lock($vmid, "create");
7139     }
7140 }
7141
7142 sub pbs_live_restore {
7143     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7144
7145     print "starting VM for live-restore\n";
7146     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7147
7148     my $pbs_backing = {};
7149     for my $ds (keys %$restored_disks) {
7150         $ds =~ m/^drive-(.*)$/;
7151         my $confname = $1;
7152         $pbs_backing->{$confname} = {
7153             repository => $opts->{repo},
7154             snapshot => $opts->{snapshot},
7155             archive => "$ds.img.fidx",
7156         };
7157         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7158         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7159
7160         my $drive = parse_drive($confname, $conf->{$confname});
7161         print "restoring '$ds' to '$drive->{file}'\n";
7162     }
7163
7164     my $drives_streamed = 0;
7165     eval {
7166         # make sure HA doesn't interrupt our restore by stopping the VM
7167         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7168             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7169         }
7170
7171         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7172         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7173         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7174
7175         my $qmeventd_fd = register_qmeventd_handle($vmid);
7176
7177         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7178         # this will effectively collapse the backing image chain consisting of
7179         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7180         # removes itself once all backing images vanish with 'auto-remove=on')
7181         my $jobs = {};
7182         for my $ds (sort keys %$restored_disks) {
7183             my $job_id = "restore-$ds";
7184             mon_cmd($vmid, 'block-stream',
7185                 'job-id' => $job_id,
7186                 device => "$ds",
7187             );
7188             $jobs->{$job_id} = {};
7189         }
7190
7191         mon_cmd($vmid, 'cont');
7192         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7193
7194         print "restore-drive jobs finished successfully, removing all tracking block devices"
7195             ." to disconnect from Proxmox Backup Server\n";
7196
7197         for my $ds (sort keys %$restored_disks) {
7198             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7199         }
7200
7201         close($qmeventd_fd);
7202     };
7203
7204     my $err = $@;
7205
7206     if ($err) {
7207         warn "An error occurred during live-restore: $err\n";
7208         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7209         die "live-restore failed\n";
7210     }
7211 }
7212
7213 sub restore_vma_archive {
7214     my ($archive, $vmid, $user, $opts, $comp) = @_;
7215
7216     my $readfrom = $archive;
7217
7218     my $cfg = PVE::Storage::config();
7219     my $commands = [];
7220     my $bwlimit = $opts->{bwlimit};
7221
7222     my $dbg_cmdstring = '';
7223     my $add_pipe = sub {
7224         my ($cmd) = @_;
7225         push @$commands, $cmd;
7226         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7227         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7228         $readfrom = '-';
7229     };
7230
7231     my $input = undef;
7232     if ($archive eq '-') {
7233         $input = '<&STDIN';
7234     } else {
7235         # If we use a backup from a PVE defined storage we also consider that
7236         # storage's rate limit:
7237         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7238         if (defined($volid)) {
7239             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7240             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7241             if ($readlimit) {
7242                 print STDERR "applying read rate limit: $readlimit\n";
7243                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7244                 $add_pipe->($cstream);
7245             }
7246         }
7247     }
7248
7249     if ($comp) {
7250         my $info = PVE::Storage::decompressor_info('vma', $comp);
7251         my $cmd = $info->{decompressor};
7252         push @$cmd, $readfrom;
7253         $add_pipe->($cmd);
7254     }
7255
7256     my $tmpdir = "/var/tmp/vzdumptmp$$";
7257     rmtree $tmpdir;
7258
7259     # disable interrupts (always do cleanups)
7260     local $SIG{INT} =
7261         local $SIG{TERM} =
7262         local $SIG{QUIT} =
7263         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7264
7265     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7266     POSIX::mkfifo($mapfifo, 0600);
7267     my $fifofh;
7268     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7269
7270     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7271
7272     my $oldtimeout;
7273     my $timeout = 5;
7274
7275     my $devinfo = {}; # info about drives included in backup
7276     my $virtdev_hash = {}; # info about allocated drives
7277
7278     my $rpcenv = PVE::RPCEnvironment::get();
7279
7280     my $conffile = PVE::QemuConfig->config_file($vmid);
7281
7282     # Note: $oldconf is undef if VM does not exist
7283     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7284     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7285     my $new_conf_raw = '';
7286
7287     my %storage_limits;
7288
7289     my $print_devmap = sub {
7290         my $cfgfn = "$tmpdir/qemu-server.conf";
7291
7292         # we can read the config - that is already extracted
7293         my $fh = IO::File->new($cfgfn, "r") ||
7294             die "unable to read qemu-server.conf - $!\n";
7295
7296         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7297         if (-f $fwcfgfn) {
7298             my $pve_firewall_dir = '/etc/pve/firewall';
7299             mkdir $pve_firewall_dir; # make sure the dir exists
7300             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7301         }
7302
7303         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7304
7305         foreach my $info (values %{$virtdev_hash}) {
7306             my $storeid = $info->{storeid};
7307             next if defined($storage_limits{$storeid});
7308
7309             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7310             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7311             $storage_limits{$storeid} = $limit * 1024;
7312         }
7313
7314         foreach my $devname (keys %$devinfo) {
7315             die "found no device mapping information for device '$devname'\n"
7316                 if !$devinfo->{$devname}->{virtdev};
7317         }
7318
7319         # create empty/temp config
7320         if ($oldconf) {
7321             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7322             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7323         }
7324
7325         # allocate volumes
7326         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7327
7328         # print restore information to $fifofh
7329         foreach my $virtdev (sort keys %$virtdev_hash) {
7330             my $d = $virtdev_hash->{$virtdev};
7331             next if $d->{is_cloudinit}; # no need to restore cloudinit
7332
7333             my $storeid = $d->{storeid};
7334             my $volid = $d->{volid};
7335
7336             my $map_opts = '';
7337             if (my $limit = $storage_limits{$storeid}) {
7338                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7339             }
7340
7341             my $write_zeros = 1;
7342             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7343                 $write_zeros = 0;
7344             }
7345
7346             my $path = PVE::Storage::path($cfg, $volid);
7347
7348             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7349
7350             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7351         }
7352
7353         $fh->seek(0, 0) || die "seek failed - $!\n";
7354
7355         my $cookie = { netcount => 0 };
7356         while (defined(my $line = <$fh>)) {
7357             $new_conf_raw .= restore_update_config_line(
7358                 $cookie,
7359                 $map,
7360                 $line,
7361                 $opts->{unique},
7362             );
7363         }
7364
7365         $fh->close();
7366     };
7367
7368     eval {
7369         # enable interrupts
7370         local $SIG{INT} =
7371             local $SIG{TERM} =
7372             local $SIG{QUIT} =
7373             local $SIG{HUP} =
7374             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7375         local $SIG{ALRM} = sub { die "got timeout\n"; };
7376
7377         $oldtimeout = alarm($timeout);
7378
7379         my $parser = sub {
7380             my $line = shift;
7381
7382             print "$line\n";
7383
7384             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7385                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7386                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7387             } elsif ($line =~ m/^CTIME: /) {
7388                 # we correctly received the vma config, so we can disable
7389                 # the timeout now for disk allocation (set to 10 minutes, so
7390                 # that we always timeout if something goes wrong)
7391                 alarm(600);
7392                 &$print_devmap();
7393                 print $fifofh "done\n";
7394                 my $tmp = $oldtimeout || 0;
7395                 $oldtimeout = undef;
7396                 alarm($tmp);
7397                 close($fifofh);
7398                 $fifofh = undef;
7399             }
7400         };
7401
7402         print "restore vma archive: $dbg_cmdstring\n";
7403         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7404     };
7405     my $err = $@;
7406
7407     alarm($oldtimeout) if $oldtimeout;
7408
7409     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7410
7411     close($fifofh) if $fifofh;
7412     unlink $mapfifo;
7413     rmtree $tmpdir;
7414
7415     if ($err) {
7416         $restore_destroy_volumes->($cfg, $virtdev_hash);
7417         die $err;
7418     }
7419
7420     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7421     PVE::QemuConfig->write_config($vmid, $new_conf);
7422
7423     eval { rescan($vmid, 1); };
7424     warn $@ if $@;
7425
7426     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7427 }
7428
7429 sub restore_tar_archive {
7430     my ($archive, $vmid, $user, $opts) = @_;
7431
7432     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7433         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7434         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7435     }
7436
7437     if ($archive ne '-') {
7438         my $firstfile = tar_archive_read_firstfile($archive);
7439         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7440             if $firstfile ne 'qemu-server.conf';
7441     }
7442
7443     my $storecfg = PVE::Storage::config();
7444
7445     # avoid zombie disks when restoring over an existing VM -> cleanup first
7446     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7447     # skiplock=1 because qmrestore has set the 'create' lock itself already
7448     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7449     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7450
7451     my $tocmd = "/usr/lib/qemu-server/qmextract";
7452
7453     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7454     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7455     $tocmd .= ' --prealloc' if $opts->{prealloc};
7456     $tocmd .= ' --info' if $opts->{info};
7457
7458     # tar option "xf" does not autodetect compression when read from STDIN,
7459     # so we pipe to zcat
7460     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7461         PVE::Tools::shellquote("--to-command=$tocmd");
7462
7463     my $tmpdir = "/var/tmp/vzdumptmp$$";
7464     mkpath $tmpdir;
7465
7466     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7467     local $ENV{VZDUMP_VMID} = $vmid;
7468     local $ENV{VZDUMP_USER} = $user;
7469
7470     my $conffile = PVE::QemuConfig->config_file($vmid);
7471     my $new_conf_raw = '';
7472
7473     # disable interrupts (always do cleanups)
7474     local $SIG{INT} =
7475         local $SIG{TERM} =
7476         local $SIG{QUIT} =
7477         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7478
7479     eval {
7480         # enable interrupts
7481         local $SIG{INT} =
7482             local $SIG{TERM} =
7483             local $SIG{QUIT} =
7484             local $SIG{HUP} =
7485             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7486
7487         if ($archive eq '-') {
7488             print "extracting archive from STDIN\n";
7489             run_command($cmd, input => "<&STDIN");
7490         } else {
7491             print "extracting archive '$archive'\n";
7492             run_command($cmd);
7493         }
7494
7495         return if $opts->{info};
7496
7497         # read new mapping
7498         my $map = {};
7499         my $statfile = "$tmpdir/qmrestore.stat";
7500         if (my $fd = IO::File->new($statfile, "r")) {
7501             while (defined (my $line = <$fd>)) {
7502                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7503                     $map->{$1} = $2 if $1;
7504                 } else {
7505                     print STDERR "unable to parse line in statfile - $line\n";
7506                 }
7507             }
7508             $fd->close();
7509         }
7510
7511         my $confsrc = "$tmpdir/qemu-server.conf";
7512
7513         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7514
7515         my $cookie = { netcount => 0 };
7516         while (defined (my $line = <$srcfd>)) {
7517             $new_conf_raw .= restore_update_config_line(
7518                 $cookie,
7519                 $map,
7520                 $line,
7521                 $opts->{unique},
7522             );
7523         }
7524
7525         $srcfd->close();
7526     };
7527     if (my $err = $@) {
7528         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7529         die $err;
7530     }
7531
7532     rmtree $tmpdir;
7533
7534     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7535
7536     PVE::Cluster::cfs_update(); # make sure we read new file
7537
7538     eval { rescan($vmid, 1); };
7539     warn $@ if $@;
7540 };
7541
7542 sub foreach_storage_used_by_vm {
7543     my ($conf, $func) = @_;
7544
7545     my $sidhash = {};
7546
7547     PVE::QemuConfig->foreach_volume($conf, sub {
7548         my ($ds, $drive) = @_;
7549         return if drive_is_cdrom($drive);
7550
7551         my $volid = $drive->{file};
7552
7553         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7554         $sidhash->{$sid} = $sid if $sid;
7555     });
7556
7557     foreach my $sid (sort keys %$sidhash) {
7558         &$func($sid);
7559     }
7560 }
7561
7562 my $qemu_snap_storage = {
7563     rbd => 1,
7564 };
7565 sub do_snapshots_with_qemu {
7566     my ($storecfg, $volid, $deviceid) = @_;
7567
7568     return if $deviceid =~ m/tpmstate0/;
7569
7570     my $storage_name = PVE::Storage::parse_volume_id($volid);
7571     my $scfg = $storecfg->{ids}->{$storage_name};
7572     die "could not find storage '$storage_name'\n" if !defined($scfg);
7573
7574     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7575         return 1;
7576     }
7577
7578     if ($volid =~ m/\.(qcow2|qed)$/){
7579         return 1;
7580     }
7581
7582     return;
7583 }
7584
7585 sub qga_check_running {
7586     my ($vmid, $nowarn) = @_;
7587
7588     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7589     if ($@) {
7590         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7591         return 0;
7592     }
7593     return 1;
7594 }
7595
7596 sub template_create {
7597     my ($vmid, $conf, $disk) = @_;
7598
7599     my $storecfg = PVE::Storage::config();
7600
7601     PVE::QemuConfig->foreach_volume($conf, sub {
7602         my ($ds, $drive) = @_;
7603
7604         return if drive_is_cdrom($drive);
7605         return if $disk && $ds ne $disk;
7606
7607         my $volid = $drive->{file};
7608         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7609
7610         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7611         $drive->{file} = $voliddst;
7612         $conf->{$ds} = print_drive($drive);
7613         PVE::QemuConfig->write_config($vmid, $conf);
7614     });
7615 }
7616
7617 sub convert_iscsi_path {
7618     my ($path) = @_;
7619
7620     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7621         my $portal = $1;
7622         my $target = $2;
7623         my $lun = $3;
7624
7625         my $initiator_name = get_initiator_name();
7626
7627         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7628                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7629     }
7630
7631     die "cannot convert iscsi path '$path', unkown format\n";
7632 }
7633
7634 sub qemu_img_convert {
7635     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7636
7637     my $storecfg = PVE::Storage::config();
7638     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7639     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7640
7641     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7642
7643     my $cachemode;
7644     my $src_path;
7645     my $src_is_iscsi = 0;
7646     my $src_format;
7647
7648     if ($src_storeid) {
7649         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7650         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7651         $src_format = qemu_img_format($src_scfg, $src_volname);
7652         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7653         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7654         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7655     } elsif (-f $src_volid || -b $src_volid) {
7656         $src_path = $src_volid;
7657         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7658             $src_format = $1;
7659         }
7660     }
7661
7662     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7663
7664     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7665     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7666     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7667     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7668
7669     my $cmd = [];
7670     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7671     push @$cmd, '-l', "snapshot.name=$snapname"
7672         if $snapname && $src_format && $src_format eq "qcow2";
7673     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7674     push @$cmd, '-T', $cachemode if defined($cachemode);
7675
7676     if ($src_is_iscsi) {
7677         push @$cmd, '--image-opts';
7678         $src_path = convert_iscsi_path($src_path);
7679     } elsif ($src_format) {
7680         push @$cmd, '-f', $src_format;
7681     }
7682
7683     if ($dst_is_iscsi) {
7684         push @$cmd, '--target-image-opts';
7685         $dst_path = convert_iscsi_path($dst_path);
7686     } else {
7687         push @$cmd, '-O', $dst_format;
7688     }
7689
7690     push @$cmd, $src_path;
7691
7692     if (!$dst_is_iscsi && $is_zero_initialized) {
7693         push @$cmd, "zeroinit:$dst_path";
7694     } else {
7695         push @$cmd, $dst_path;
7696     }
7697
7698     my $parser = sub {
7699         my $line = shift;
7700         if($line =~ m/\((\S+)\/100\%\)/){
7701             my $percent = $1;
7702             my $transferred = int($size * $percent / 100);
7703             my $total_h = render_bytes($size, 1);
7704             my $transferred_h = render_bytes($transferred, 1);
7705
7706             print "transferred $transferred_h of $total_h ($percent%)\n";
7707         }
7708
7709     };
7710
7711     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7712     my $err = $@;
7713     die "copy failed: $err" if $err;
7714 }
7715
7716 sub qemu_img_format {
7717     my ($scfg, $volname) = @_;
7718
7719     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7720         return $1;
7721     } else {
7722         return "raw";
7723     }
7724 }
7725
7726 sub qemu_drive_mirror {
7727     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7728
7729     $jobs = {} if !$jobs;
7730
7731     my $qemu_target;
7732     my $format;
7733     $jobs->{"drive-$drive"} = {};
7734
7735     if ($dst_volid =~ /^nbd:/) {
7736         $qemu_target = $dst_volid;
7737         $format = "nbd";
7738     } else {
7739         my $storecfg = PVE::Storage::config();
7740         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7741
7742         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7743
7744         $format = qemu_img_format($dst_scfg, $dst_volname);
7745
7746         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7747
7748         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7749     }
7750
7751     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7752     $opts->{format} = $format if $format;
7753
7754     if (defined($src_bitmap)) {
7755         $opts->{sync} = 'incremental';
7756         $opts->{bitmap} = $src_bitmap;
7757         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7758     }
7759
7760     if (defined($bwlimit)) {
7761         $opts->{speed} = $bwlimit * 1024;
7762         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7763     } else {
7764         print "drive mirror is starting for drive-$drive\n";
7765     }
7766
7767     # if a job already runs for this device we get an error, catch it for cleanup
7768     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7769     if (my $err = $@) {
7770         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7771         warn "$@\n" if $@;
7772         die "mirroring error: $err\n";
7773     }
7774
7775     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7776 }
7777
7778 # $completion can be either
7779 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7780 # 'cancel': wait until all jobs are ready, block-job-cancel them
7781 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7782 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7783 sub qemu_drive_mirror_monitor {
7784     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7785
7786     $completion //= 'complete';
7787     $op //= "mirror";
7788
7789     eval {
7790         my $err_complete = 0;
7791
7792         my $starttime = time ();
7793         while (1) {
7794             die "block job ('$op') timed out\n" if $err_complete > 300;
7795
7796             my $stats = mon_cmd($vmid, "query-block-jobs");
7797             my $ctime = time();
7798
7799             my $running_jobs = {};
7800             for my $stat (@$stats) {
7801                 next if $stat->{type} ne $op;
7802                 $running_jobs->{$stat->{device}} = $stat;
7803             }
7804
7805             my $readycounter = 0;
7806
7807             for my $job_id (sort keys %$jobs) {
7808                 my $job = $running_jobs->{$job_id};
7809
7810                 my $vanished = !defined($job);
7811                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7812                 if($complete || ($vanished && $completion eq 'auto')) {
7813                     print "$job_id: $op-job finished\n";
7814                     delete $jobs->{$job_id};
7815                     next;
7816                 }
7817
7818                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7819
7820                 my $busy = $job->{busy};
7821                 my $ready = $job->{ready};
7822                 if (my $total = $job->{len}) {
7823                     my $transferred = $job->{offset} || 0;
7824                     my $remaining = $total - $transferred;
7825                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7826
7827                     my $duration = $ctime - $starttime;
7828                     my $total_h = render_bytes($total, 1);
7829                     my $transferred_h = render_bytes($transferred, 1);
7830
7831                     my $status = sprintf(
7832                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7833                         render_duration($duration),
7834                     );
7835
7836                     if ($ready) {
7837                         if ($busy) {
7838                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7839                         } else {
7840                             $status .= ", ready";
7841                         }
7842                     }
7843                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7844                     $jobs->{$job_id}->{ready} = $ready;
7845                 }
7846
7847                 $readycounter++ if $job->{ready};
7848             }
7849
7850             last if scalar(keys %$jobs) == 0;
7851
7852             if ($readycounter == scalar(keys %$jobs)) {
7853                 print "all '$op' jobs are ready\n";
7854
7855                 # do the complete later (or has already been done)
7856                 last if $completion eq 'skip' || $completion eq 'auto';
7857
7858                 if ($vmiddst && $vmiddst != $vmid) {
7859                     my $agent_running = $qga && qga_check_running($vmid);
7860                     if ($agent_running) {
7861                         print "freeze filesystem\n";
7862                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7863                         warn $@ if $@;
7864                     } else {
7865                         print "suspend vm\n";
7866                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7867                         warn $@ if $@;
7868                     }
7869
7870                     # if we clone a disk for a new target vm, we don't switch the disk
7871                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7872
7873                     if ($agent_running) {
7874                         print "unfreeze filesystem\n";
7875                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7876                         warn $@ if $@;
7877                     } else {
7878                         print "resume vm\n";
7879                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7880                         warn $@ if $@;
7881                     }
7882
7883                     last;
7884                 } else {
7885
7886                     for my $job_id (sort keys %$jobs) {
7887                         # try to switch the disk if source and destination are on the same guest
7888                         print "$job_id: Completing block job_id...\n";
7889
7890                         my $op;
7891                         if ($completion eq 'complete') {
7892                             $op = 'block-job-complete';
7893                         } elsif ($completion eq 'cancel') {
7894                             $op = 'block-job-cancel';
7895                         } else {
7896                             die "invalid completion value: $completion\n";
7897                         }
7898                         eval { mon_cmd($vmid, $op, device => $job_id) };
7899                         if ($@ =~ m/cannot be completed/) {
7900                             print "$job_id: block job cannot be completed, trying again.\n";
7901                             $err_complete++;
7902                         }else {
7903                             print "$job_id: Completed successfully.\n";
7904                             $jobs->{$job_id}->{complete} = 1;
7905                         }
7906                     }
7907                 }
7908             }
7909             sleep 1;
7910         }
7911     };
7912     my $err = $@;
7913
7914     if ($err) {
7915         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7916         die "block job ($op) error: $err";
7917     }
7918 }
7919
7920 sub qemu_blockjobs_cancel {
7921     my ($vmid, $jobs) = @_;
7922
7923     foreach my $job (keys %$jobs) {
7924         print "$job: Cancelling block job\n";
7925         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7926         $jobs->{$job}->{cancel} = 1;
7927     }
7928
7929     while (1) {
7930         my $stats = mon_cmd($vmid, "query-block-jobs");
7931
7932         my $running_jobs = {};
7933         foreach my $stat (@$stats) {
7934             $running_jobs->{$stat->{device}} = $stat;
7935         }
7936
7937         foreach my $job (keys %$jobs) {
7938
7939             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7940                 print "$job: Done.\n";
7941                 delete $jobs->{$job};
7942             }
7943         }
7944
7945         last if scalar(keys %$jobs) == 0;
7946
7947         sleep 1;
7948     }
7949 }
7950
7951 sub clone_disk {
7952     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7953
7954     my ($vmid, $running) = $source->@{qw(vmid running)};
7955     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7956
7957     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7958     my ($storage, $format) = $dest->@{qw(storage format)};
7959
7960     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7961
7962     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7963         die "cloning from/to EFI disk requires EFI disk\n"
7964             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7965         die "cloning from/to TPM state requires TPM state\n"
7966             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7967
7968         # This would lead to two device nodes in QEMU pointing to the same backing image!
7969         die "cannot change drive name when cloning disk from/to the same VM\n"
7970             if $use_drive_mirror && $vmid == $newvmid;
7971     }
7972
7973     die "cannot move TPM state while VM is running\n"
7974         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7975
7976     my $newvolid;
7977
7978     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7979     print "$src_drivename " if $src_drivename;
7980     print "($drive->{file})\n";
7981
7982     if (!$full) {
7983         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7984         push @$newvollist, $newvolid;
7985     } else {
7986
7987         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7988         $storeid = $storage if $storage;
7989
7990         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7991
7992         my $name = undef;
7993         my $size = undef;
7994         if (drive_is_cloudinit($drive)) {
7995             $name = "vm-$newvmid-cloudinit";
7996             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7997             if ($scfg->{path}) {
7998                 $name .= ".$dst_format";
7999             }
8000             $snapname = undef;
8001             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8002         } elsif ($dst_drivename eq 'efidisk0') {
8003             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8004         } elsif ($dst_drivename eq 'tpmstate0') {
8005             $dst_format = 'raw';
8006             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8007         } else {
8008             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8009         }
8010         $newvolid = PVE::Storage::vdisk_alloc(
8011             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8012         );
8013         push @$newvollist, $newvolid;
8014
8015         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8016
8017         if (drive_is_cloudinit($drive)) {
8018             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8019             # if this is the case, we have to complete any block-jobs still there from
8020             # previous drive-mirrors
8021             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8022                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8023             }
8024             goto no_data_clone;
8025         }
8026
8027         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8028         if ($use_drive_mirror) {
8029             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8030                 $completion, $qga, $bwlimit);
8031         } else {
8032             # TODO: handle bwlimits
8033             if ($dst_drivename eq 'efidisk0') {
8034                 # the relevant data on the efidisk may be smaller than the source
8035                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8036                 # that is given by the OVMF_VARS.fd
8037                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8038                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8039
8040                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8041
8042                 # better for Ceph if block size is not too small, see bug #3324
8043                 my $bs = 1024*1024;
8044
8045                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8046
8047                 if ($src_format eq 'qcow2' && $snapname) {
8048                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8049                         if !min_version(kvm_user_version(), 6, 2);
8050                     push $cmd->@*, '-l', $snapname;
8051                 }
8052                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8053                 run_command($cmd);
8054             } else {
8055                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
8056             }
8057         }
8058     }
8059
8060 no_data_clone:
8061     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8062
8063     my $disk = dclone($drive);
8064     delete $disk->{format};
8065     $disk->{file} = $newvolid;
8066     $disk->{size} = $size if defined($size);
8067
8068     return $disk;
8069 }
8070
8071 sub get_running_qemu_version {
8072     my ($vmid) = @_;
8073     my $res = mon_cmd($vmid, "query-version");
8074     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8075 }
8076
8077 sub qemu_use_old_bios_files {
8078     my ($machine_type) = @_;
8079
8080     return if !$machine_type;
8081
8082     my $use_old_bios_files = undef;
8083
8084     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8085         $machine_type = $1;
8086         $use_old_bios_files = 1;
8087     } else {
8088         my $version = extract_version($machine_type, kvm_user_version());
8089         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8090         # load new efi bios files on migration. So this hack is required to allow
8091         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8092         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8093         $use_old_bios_files = !min_version($version, 2, 4);
8094     }
8095
8096     return ($use_old_bios_files, $machine_type);
8097 }
8098
8099 sub get_efivars_size {
8100     my ($conf, $efidisk) = @_;
8101
8102     my $arch = get_vm_arch($conf);
8103     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8104     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8105     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8106     return -s $ovmf_vars;
8107 }
8108
8109 sub update_efidisk_size {
8110     my ($conf) = @_;
8111
8112     return if !defined($conf->{efidisk0});
8113
8114     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8115     $disk->{size} = get_efivars_size($conf);
8116     $conf->{efidisk0} = print_drive($disk);
8117
8118     return;
8119 }
8120
8121 sub update_tpmstate_size {
8122     my ($conf) = @_;
8123
8124     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8125     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8126     $conf->{tpmstate0} = print_drive($disk);
8127 }
8128
8129 sub create_efidisk($$$$$$$) {
8130     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8131
8132     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8133
8134     my $vars_size_b = -s $ovmf_vars;
8135     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8136     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8137     PVE::Storage::activate_volumes($storecfg, [$volid]);
8138
8139     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8140     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8141
8142     return ($volid, $size/1024);
8143 }
8144
8145 sub vm_iothreads_list {
8146     my ($vmid) = @_;
8147
8148     my $res = mon_cmd($vmid, 'query-iothreads');
8149
8150     my $iothreads = {};
8151     foreach my $iothread (@$res) {
8152         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8153     }
8154
8155     return $iothreads;
8156 }
8157
8158 sub scsihw_infos {
8159     my ($conf, $drive) = @_;
8160
8161     my $maxdev = 0;
8162
8163     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8164         $maxdev = 7;
8165     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8166         $maxdev = 1;
8167     } else {
8168         $maxdev = 256;
8169     }
8170
8171     my $controller = int($drive->{index} / $maxdev);
8172     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8173         ? "virtioscsi"
8174         : "scsihw";
8175
8176     return ($maxdev, $controller, $controller_prefix);
8177 }
8178
8179 sub resolve_dst_disk_format {
8180         my ($storecfg, $storeid, $src_volname, $format) = @_;
8181         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8182
8183         if (!$format) {
8184             # if no target format is specified, use the source disk format as hint
8185             if ($src_volname) {
8186                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8187                 $format = qemu_img_format($scfg, $src_volname);
8188             } else {
8189                 return $defFormat;
8190             }
8191         }
8192
8193         # test if requested format is supported - else use default
8194         my $supported = grep { $_ eq $format } @$validFormats;
8195         $format = $defFormat if !$supported;
8196         return $format;
8197 }
8198
8199 # NOTE: if this logic changes, please update docs & possibly gui logic
8200 sub find_vmstate_storage {
8201     my ($conf, $storecfg) = @_;
8202
8203     # first, return storage from conf if set
8204     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8205
8206     my ($target, $shared, $local);
8207
8208     foreach_storage_used_by_vm($conf, sub {
8209         my ($sid) = @_;
8210         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8211         my $dst = $scfg->{shared} ? \$shared : \$local;
8212         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8213     });
8214
8215     # second, use shared storage where VM has at least one disk
8216     # third, use local storage where VM has at least one disk
8217     # fall back to local storage
8218     $target = $shared // $local // 'local';
8219
8220     return $target;
8221 }
8222
8223 sub generate_uuid {
8224     my ($uuid, $uuid_str);
8225     UUID::generate($uuid);
8226     UUID::unparse($uuid, $uuid_str);
8227     return $uuid_str;
8228 }
8229
8230 sub generate_smbios1_uuid {
8231     return "uuid=".generate_uuid();
8232 }
8233
8234 sub nbd_stop {
8235     my ($vmid) = @_;
8236
8237     mon_cmd($vmid, 'nbd-server-stop');
8238 }
8239
8240 sub create_reboot_request {
8241     my ($vmid) = @_;
8242     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8243         or die "failed to create reboot trigger file: $!\n";
8244     close($fh);
8245 }
8246
8247 sub clear_reboot_request {
8248     my ($vmid) = @_;
8249     my $path = "/run/qemu-server/$vmid.reboot";
8250     my $res = 0;
8251
8252     $res = unlink($path);
8253     die "could not remove reboot request for $vmid: $!"
8254         if !$res && $! != POSIX::ENOENT;
8255
8256     return $res;
8257 }
8258
8259 sub bootorder_from_legacy {
8260     my ($conf, $bootcfg) = @_;
8261
8262     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8263     my $bootindex_hash = {};
8264     my $i = 1;
8265     foreach my $o (split(//, $boot)) {
8266         $bootindex_hash->{$o} = $i*100;
8267         $i++;
8268     }
8269
8270     my $bootorder = {};
8271
8272     PVE::QemuConfig->foreach_volume($conf, sub {
8273         my ($ds, $drive) = @_;
8274
8275         if (drive_is_cdrom ($drive, 1)) {
8276             if ($bootindex_hash->{d}) {
8277                 $bootorder->{$ds} = $bootindex_hash->{d};
8278                 $bootindex_hash->{d} += 1;
8279             }
8280         } elsif ($bootindex_hash->{c}) {
8281             $bootorder->{$ds} = $bootindex_hash->{c}
8282                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8283             $bootindex_hash->{c} += 1;
8284         }
8285     });
8286
8287     if ($bootindex_hash->{n}) {
8288         for (my $i = 0; $i < $MAX_NETS; $i++) {
8289             my $netname = "net$i";
8290             next if !$conf->{$netname};
8291             $bootorder->{$netname} = $bootindex_hash->{n};
8292             $bootindex_hash->{n} += 1;
8293         }
8294     }
8295
8296     return $bootorder;
8297 }
8298
8299 # Generate default device list for 'boot: order=' property. Matches legacy
8300 # default boot order, but with explicit device names. This is important, since
8301 # the fallback for when neither 'order' nor the old format is specified relies
8302 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8303 sub get_default_bootdevices {
8304     my ($conf) = @_;
8305
8306     my @ret = ();
8307
8308     # harddisk
8309     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8310     push @ret, $first if $first;
8311
8312     # cdrom
8313     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8314     push @ret, $first if $first;
8315
8316     # network
8317     for (my $i = 0; $i < $MAX_NETS; $i++) {
8318         my $netname = "net$i";
8319         next if !$conf->{$netname};
8320         push @ret, $netname;
8321         last;
8322     }
8323
8324     return \@ret;
8325 }
8326
8327 sub device_bootorder {
8328     my ($conf) = @_;
8329
8330     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8331
8332     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8333
8334     my $bootorder = {};
8335     if (!defined($boot) || $boot->{legacy}) {
8336         $bootorder = bootorder_from_legacy($conf, $boot);
8337     } elsif ($boot->{order}) {
8338         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8339         for my $dev (PVE::Tools::split_list($boot->{order})) {
8340             $bootorder->{$dev} = $i++;
8341         }
8342     }
8343
8344     return $bootorder;
8345 }
8346
8347 sub register_qmeventd_handle {
8348     my ($vmid) = @_;
8349
8350     my $fh;
8351     my $peer = "/var/run/qmeventd.sock";
8352     my $count = 0;
8353
8354     for (;;) {
8355         $count++;
8356         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8357         last if $fh;
8358         if ($! != EINTR && $! != EAGAIN) {
8359             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8360         }
8361         if ($count > 4) {
8362             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8363               . "after $count retries\n";
8364         }
8365         usleep(25000);
8366     }
8367
8368     # send handshake to mark VM as backing up
8369     print $fh to_json({vzdump => {vmid => "$vmid"}});
8370
8371     # return handle to be closed later when inhibit is no longer required
8372     return $fh;
8373 }
8374
8375 # bash completion helper
8376
8377 sub complete_backup_archives {
8378     my ($cmdname, $pname, $cvalue) = @_;
8379
8380     my $cfg = PVE::Storage::config();
8381
8382     my $storeid;
8383
8384     if ($cvalue =~ m/^([^:]+):/) {
8385         $storeid = $1;
8386     }
8387
8388     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8389
8390     my $res = [];
8391     foreach my $id (keys %$data) {
8392         foreach my $item (@{$data->{$id}}) {
8393             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8394             push @$res, $item->{volid} if defined($item->{volid});
8395         }
8396     }
8397
8398     return $res;
8399 }
8400
8401 my $complete_vmid_full = sub {
8402     my ($running) = @_;
8403
8404     my $idlist = vmstatus();
8405
8406     my $res = [];
8407
8408     foreach my $id (keys %$idlist) {
8409         my $d = $idlist->{$id};
8410         if (defined($running)) {
8411             next if $d->{template};
8412             next if $running && $d->{status} ne 'running';
8413             next if !$running && $d->{status} eq 'running';
8414         }
8415         push @$res, $id;
8416
8417     }
8418     return $res;
8419 };
8420
8421 sub complete_vmid {
8422     return &$complete_vmid_full();
8423 }
8424
8425 sub complete_vmid_stopped {
8426     return &$complete_vmid_full(0);
8427 }
8428
8429 sub complete_vmid_running {
8430     return &$complete_vmid_full(1);
8431 }
8432
8433 sub complete_storage {
8434
8435     my $cfg = PVE::Storage::config();
8436     my $ids = $cfg->{ids};
8437
8438     my $res = [];
8439     foreach my $sid (keys %$ids) {
8440         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8441         next if !$ids->{$sid}->{content}->{images};
8442         push @$res, $sid;
8443     }
8444
8445     return $res;
8446 }
8447
8448 sub complete_migration_storage {
8449     my ($cmd, $param, $current_value, $all_args) = @_;
8450
8451     my $targetnode = @$all_args[1];
8452
8453     my $cfg = PVE::Storage::config();
8454     my $ids = $cfg->{ids};
8455
8456     my $res = [];
8457     foreach my $sid (keys %$ids) {
8458         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8459         next if !$ids->{$sid}->{content}->{images};
8460         push @$res, $sid;
8461     }
8462
8463     return $res;
8464 }
8465
8466 sub vm_is_paused {
8467     my ($vmid) = @_;
8468     my $qmpstatus = eval {
8469         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8470         mon_cmd($vmid, "query-status");
8471     };
8472     warn "$@\n" if $@;
8473     return $qmpstatus && $qmpstatus->{status} eq "paused";
8474 }
8475
8476 sub check_volume_storage_type {
8477     my ($storecfg, $vol) = @_;
8478
8479     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8480     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8481     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8482
8483     die "storage '$storeid' does not support content-type '$vtype'\n"
8484         if !$scfg->{content}->{$vtype};
8485
8486     return 1;
8487 }
8488
8489 sub add_nets_bridge_fdb {
8490     my ($conf, $vmid) = @_;
8491
8492     for my $opt (keys %$conf) {
8493         next if $opt !~ m/^net(\d+)$/;
8494         my $iface = "tap${vmid}i$1";
8495         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8496         my $net = parse_net($conf->{$opt}, 1) or next;
8497
8498         my $mac = $net->{macaddr};
8499         if (!$mac) {
8500             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8501                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8502             next;
8503         }
8504
8505         my $bridge = $net->{bridge};
8506         if ($have_sdn) {
8507             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8508         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8509             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8510         }
8511     }
8512 }
8513
8514 sub del_nets_bridge_fdb {
8515     my ($conf, $vmid) = @_;
8516
8517     for my $opt (keys %$conf) {
8518         next if $opt !~ m/^net(\d+)$/;
8519         my $iface = "tap${vmid}i$1";
8520
8521         my $net = parse_net($conf->{$opt}) or next;
8522         my $mac = $net->{macaddr} or next;
8523
8524         my $bridge = $net->{bridge};
8525         if ($have_sdn) {
8526             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8527         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8528             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8529         }
8530     }
8531 }
8532
8533 1;