PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory qw(get_current_memory);
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  62 use PVE::QemuServer::USB;
  63
  64 my $have_sdn;
  65 eval {
  66     require PVE::Network::SDN::Zones;
  67     require PVE::Network::SDN::Vnets;
  68     $have_sdn = 1;
  69 };
  70
  71 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  72 my $OVMF = {
  73     x86_64 => {
  74         '4m-no-smm' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  77         ],
  78         '4m-no-smm-ms' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  81         ],
  82         '4m' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  85         ],
  86         '4m-ms' => [
  87             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  89         ],
  90         # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
  91         # anymore. how can we deperacate this sanely without breaking existing instances, or using
  92         # older backups and snapshot?
  93         default => [
  94             "$EDK2_FW_BASE/OVMF_CODE.fd",
  95             "$EDK2_FW_BASE/OVMF_VARS.fd",
  96         ],
  97     },
  98     aarch64 => {
  99         default => [
 100             "$EDK2_FW_BASE/AAVMF_CODE.fd",
 101             "$EDK2_FW_BASE/AAVMF_VARS.fd",
 102         ],
 103     },
 104 };
 105
 106 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 107
 108 # Note about locking: we use flock on the config file protect against concurent actions.
 109 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 110 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 111 # But you can ignore this kind of lock with the --skiplock flag.
 112
 113 cfs_register_file(
 114     '/qemu-server/',
 115     \&parse_vm_config,
 116     \&write_vm_config
 117 );
 118
 119 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 120     description => "Some command save/restore state from this location.",
 121     type => 'string',
 122     maxLength => 128,
 123     optional => 1,
 124 });
 125
 126 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 127         description => "Specifies the QEMU machine type.",
 128         type => 'string',
 129         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 130         maxLength => 40,
 131         optional => 1,
 132 });
 133
 134 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 135 my $nodename_cache;
 136 sub nodename {
 137     $nodename_cache //= PVE::INotify::nodename();
 138     return $nodename_cache;
 139 }
 140
 141 my $watchdog_fmt = {
 142     model => {
 143         default_key => 1,
 144         type => 'string',
 145         enum => [qw(i6300esb ib700)],
 146         description => "Watchdog type to emulate.",
 147         default => 'i6300esb',
 148         optional => 1,
 149     },
 150     action => {
 151         type => 'string',
 152         enum => [qw(reset shutdown poweroff pause debug none)],
 153         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 154         optional => 1,
 155     },
 156 };
 157 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 158
 159 my $agent_fmt = {
 160     enabled => {
 161         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 162         type => 'boolean',
 163         default => 0,
 164         default_key => 1,
 165     },
 166     fstrim_cloned_disks => {
 167         description => "Run fstrim after moving a disk or migrating the VM.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0,
 171     },
 172     'freeze-fs-on-backup' => {
 173         description => "Freeze/thaw guest filesystems on backup for consistency.",
 174         type => 'boolean',
 175         optional => 1,
 176         default => 1,
 177     },
 178     type => {
 179         description => "Select the agent type",
 180         type => 'string',
 181         default => 'virtio',
 182         optional => 1,
 183         enum => [qw(virtio isa)],
 184     },
 185 };
 186
 187 my $vga_fmt = {
 188     type => {
 189         description => "Select the VGA type.",
 190         type => 'string',
 191         default => 'std',
 192         optional => 1,
 193         default_key => 1,
 194         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 195     },
 196     memory => {
 197         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 198         type => 'integer',
 199         optional => 1,
 200         minimum => 4,
 201         maximum => 512,
 202     },
 203     clipboard => {
 204         description => 'Enable a specific clipboard. If not set, depending on the display type the'
 205             .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
 206         type => 'string',
 207         enum => ['vnc'],
 208         optional => 1,
 209     },
 210 };
 211
 212 my $ivshmem_fmt = {
 213     size => {
 214         type => 'integer',
 215         minimum => 1,
 216         description => "The size of the file in MB.",
 217     },
 218     name => {
 219         type => 'string',
 220         pattern => '[a-zA-Z0-9\-]+',
 221         optional => 1,
 222         format_description => 'string',
 223         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 224     },
 225 };
 226
 227 my $audio_fmt = {
 228     device => {
 229         type => 'string',
 230         enum => [qw(ich9-intel-hda intel-hda AC97)],
 231         description =>  "Configure an audio device."
 232     },
 233     driver =>  {
 234         type => 'string',
 235         enum => ['spice', 'none'],
 236         default => 'spice',
 237         optional => 1,
 238         description => "Driver backend for the audio device."
 239     },
 240 };
 241
 242 my $spice_enhancements_fmt = {
 243     foldersharing => {
 244         type => 'boolean',
 245         optional => 1,
 246         default => '0',
 247         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 248     },
 249     videostreaming =>  {
 250         type => 'string',
 251         enum => ['off', 'all', 'filter'],
 252         default => 'off',
 253         optional => 1,
 254         description => "Enable video streaming. Uses compression for detected video streams."
 255     },
 256 };
 257
 258 my $rng_fmt = {
 259     source => {
 260         type => 'string',
 261         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 262         default_key => 1,
 263         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 264             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 265             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 266             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 267             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 268             ." a hardware RNG from the host.",
 269     },
 270     max_bytes => {
 271         type => 'integer',
 272         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 273             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 274             ." `0` to disable limiting (potentially dangerous!).",
 275         optional => 1,
 276
 277         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 278         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 279         # reading from /dev/urandom
 280         default => 1024,
 281     },
 282     period => {
 283         type => 'integer',
 284         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 285             ." the guest to retrieve another 'max_bytes' of entropy.",
 286         optional => 1,
 287         default => 1000,
 288     },
 289 };
 290
 291 my $meta_info_fmt = {
 292     'ctime' => {
 293         type => 'integer',
 294         description => "The guest creation timestamp as UNIX epoch time",
 295         minimum => 0,
 296         optional => 1,
 297     },
 298     'creation-qemu' => {
 299         type => 'string',
 300         description => "The QEMU (machine) version from the time this VM was created.",
 301         pattern => '\d+(\.\d+)+',
 302         optional => 1,
 303     },
 304 };
 305
 306 my $confdesc = {
 307     onboot => {
 308         optional => 1,
 309         type => 'boolean',
 310         description => "Specifies whether a VM will be started during system bootup.",
 311         default => 0,
 312     },
 313     autostart => {
 314         optional => 1,
 315         type => 'boolean',
 316         description => "Automatic restart after crash (currently ignored).",
 317         default => 0,
 318     },
 319     hotplug => {
 320         optional => 1,
 321         type => 'string', format => 'pve-hotplug-features',
 322         description => "Selectively enable hotplug features. This is a comma separated list of"
 323             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 324             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 325             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 326             ." windows > 7.",
 327         default => 'network,disk,usb',
 328     },
 329     reboot => {
 330         optional => 1,
 331         type => 'boolean',
 332         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 333         default => 1,
 334     },
 335     lock => {
 336         optional => 1,
 337         type => 'string',
 338         description => "Lock/unlock the VM.",
 339         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 340     },
 341     cpulimit => {
 342         optional => 1,
 343         type => 'number',
 344         description => "Limit of CPU usage.",
 345         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 346             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 347         minimum => 0,
 348         maximum => 128,
 349         default => 0,
 350     },
 351     cpuunits => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 355         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 356             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 357             ." weights of all the other running VMs.",
 358         minimum => 1,
 359         maximum => 262144,
 360         default => 'cgroup v1: 1024, cgroup v2: 100',
 361     },
 362     memory => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Memory properties.",
 366         format => $PVE::QemuServer::Memory::memory_fmt
 367     },
 368     balloon => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 372         minimum => 0,
 373     },
 374     shares => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 378             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 379             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 380         minimum => 0,
 381         maximum => 50000,
 382         default => 1000,
 383     },
 384     keyboard => {
 385         optional => 1,
 386         type => 'string',
 387         description => "Keyboard layout for VNC server. This option is generally not required and"
 388         ." is often better handled from within the guest OS.",
 389         enum => PVE::Tools::kvmkeymaplist(),
 390         default => undef,
 391     },
 392     name => {
 393         optional => 1,
 394         type => 'string', format => 'dns-name',
 395         description => "Set a name for the VM. Only used on the configuration web interface.",
 396     },
 397     scsihw => {
 398         optional => 1,
 399         type => 'string',
 400         description => "SCSI controller model",
 401         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 402         default => 'lsi',
 403     },
 404     description => {
 405         optional => 1,
 406         type => 'string',
 407         description => "Description for the VM. Shown in the web-interface VM's summary."
 408             ." This is saved as comment inside the configuration file.",
 409         maxLength => 1024 * 8,
 410     },
 411     ostype => {
 412         optional => 1,
 413         type => 'string',
 414         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 415         description => "Specify guest operating system.",
 416         verbose_description => <<EODESC,
 417 Specify guest operating system. This is used to enable special
 418 optimization/features for specific operating systems:
 419
 420 [horizontal]
 421 other;; unspecified OS
 422 wxp;; Microsoft Windows XP
 423 w2k;; Microsoft Windows 2000
 424 w2k3;; Microsoft Windows 2003
 425 w2k8;; Microsoft Windows 2008
 426 wvista;; Microsoft Windows Vista
 427 win7;; Microsoft Windows 7
 428 win8;; Microsoft Windows 8/2012/2012r2
 429 win10;; Microsoft Windows 10/2016/2019
 430 win11;; Microsoft Windows 11/2022
 431 l24;; Linux 2.4 Kernel
 432 l26;; Linux 2.6 - 6.X Kernel
 433 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 434 EODESC
 435     },
 436     boot => {
 437         optional => 1,
 438         type => 'string', format => 'pve-qm-boot',
 439         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 440             ." key or 'legacy=' is deprecated.",
 441     },
 442     bootdisk => {
 443         optional => 1,
 444         type => 'string', format => 'pve-qm-bootdisk',
 445         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 446         pattern => '(ide|sata|scsi|virtio)\d+',
 447     },
 448     smp => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "The number of CPUs. Please use option -sockets instead.",
 452         minimum => 1,
 453         default => 1,
 454     },
 455     sockets => {
 456         optional => 1,
 457         type => 'integer',
 458         description => "The number of CPU sockets.",
 459         minimum => 1,
 460         default => 1,
 461     },
 462     cores => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "The number of cores per socket.",
 466         minimum => 1,
 467         default => 1,
 468     },
 469     numa => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable NUMA.",
 473         default => 0,
 474     },
 475     hugepages => {
 476         optional => 1,
 477         type => 'string',
 478         description => "Enable/disable hugepages memory.",
 479         enum => [qw(any 2 1024)],
 480     },
 481     keephugepages => {
 482         optional => 1,
 483         type => 'boolean',
 484         default => 0,
 485         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 486             ." after VM shutdown and can be used for subsequent starts.",
 487     },
 488     vcpus => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Number of hotplugged vcpus.",
 492         minimum => 1,
 493         default => 0,
 494     },
 495     acpi => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable ACPI.",
 499         default => 1,
 500     },
 501     agent => {
 502         optional => 1,
 503         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 504         type => 'string',
 505         format => $agent_fmt,
 506     },
 507     kvm => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable KVM hardware virtualization.",
 511         default => 1,
 512     },
 513     tdf => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable time drift fix.",
 517         default => 0,
 518     },
 519     localtime => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 523             ." the `ostype` indicates a Microsoft Windows OS.",
 524     },
 525     freeze => {
 526         optional => 1,
 527         type => 'boolean',
 528         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 529     },
 530     vga => {
 531         optional => 1,
 532         type => 'string', format => $vga_fmt,
 533         description => "Configure the VGA hardware.",
 534         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 535             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 536             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 537             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 538             ." display server. For win* OS you can select how many independent displays you want,"
 539             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 540             ." using a serial device as terminal.",
 541     },
 542     watchdog => {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-watchdog',
 545         description => "Create a virtual hardware watchdog device.",
 546         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 547             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 548             ." else the watchdog will reset the guest (or execute the respective action specified)",
 549     },
 550     startdate => {
 551         optional => 1,
 552         type => 'string',
 553         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 554         description => "Set the initial date of the real time clock. Valid format for date are:"
 555             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 556         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 557         default => 'now',
 558     },
 559     startup =>  get_standard_option('pve-startup-order'),
 560     template => {
 561         optional => 1,
 562         type => 'boolean',
 563         description => "Enable/disable Template.",
 564         default => 0,
 565     },
 566     args => {
 567         optional => 1,
 568         type => 'string',
 569         description => "Arbitrary arguments passed to kvm.",
 570         verbose_description => <<EODESCR,
 571 Arbitrary arguments passed to kvm, for example:
 572
 573 args: -no-reboot -smbios 'type=0,vendor=FOO'
 574
 575 NOTE: this option is for experts only.
 576 EODESCR
 577     },
 578     tablet => {
 579         optional => 1,
 580         type => 'boolean',
 581         default => 1,
 582         description => "Enable/disable the USB tablet device.",
 583         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 584             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 585             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 586             ." may consider disabling this to save some context switches. This is turned off by"
 587             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 588     },
 589     migrate_speed => {
 590         optional => 1,
 591         type => 'integer',
 592         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 593         minimum => 0,
 594         default => 0,
 595     },
 596     migrate_downtime => {
 597         optional => 1,
 598         type => 'number',
 599         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 600         minimum => 0,
 601         default => 0.1,
 602     },
 603     cdrom => {
 604         optional => 1,
 605         type => 'string', format => 'pve-qm-ide',
 606         typetext => '<volume>',
 607         description => "This is an alias for option -ide2",
 608     },
 609     cpu => {
 610         optional => 1,
 611         description => "Emulated CPU type.",
 612         type => 'string',
 613         format => 'pve-vm-cpu-conf',
 614     },
 615     parent => get_standard_option('pve-snapshot-name', {
 616         optional => 1,
 617         description => "Parent snapshot name. This is used internally, and should not be modified.",
 618     }),
 619     snaptime => {
 620         optional => 1,
 621         description => "Timestamp for snapshots.",
 622         type => 'integer',
 623         minimum => 0,
 624     },
 625     vmstate => {
 626         optional => 1,
 627         type => 'string', format => 'pve-volume-id',
 628         description => "Reference to a volume which stores the VM state. This is used internally"
 629             ." for snapshots.",
 630     },
 631     vmstatestorage => get_standard_option('pve-storage-id', {
 632         description => "Default storage for VM state volumes/files.",
 633         optional => 1,
 634     }),
 635     runningmachine => get_standard_option('pve-qemu-machine', {
 636         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 637             ." for snapshots.",
 638     }),
 639     runningcpu => {
 640         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 641             ." internally for snapshots.",
 642         optional => 1,
 643         type => 'string',
 644         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 645         format_description => 'QEMU -cpu parameter'
 646     },
 647     machine => get_standard_option('pve-qemu-machine'),
 648     arch => {
 649         description => "Virtual processor architecture. Defaults to the host.",
 650         optional => 1,
 651         type => 'string',
 652         enum => [qw(x86_64 aarch64)],
 653     },
 654     smbios1 => {
 655         description => "Specify SMBIOS type 1 fields.",
 656         type => 'string', format => 'pve-qm-smbios1',
 657         maxLength => 512,
 658         optional => 1,
 659     },
 660     protection => {
 661         optional => 1,
 662         type => 'boolean',
 663         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 664             ." remove disk operations.",
 665         default => 0,
 666     },
 667     bios => {
 668         optional => 1,
 669         type => 'string',
 670         enum => [ qw(seabios ovmf) ],
 671         description => "Select BIOS implementation.",
 672         default => 'seabios',
 673     },
 674     vmgenid => {
 675         type => 'string',
 676         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 677         format_description => 'UUID',
 678         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 679             ." to disable explicitly.",
 680         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 681             ." value identifier to the guest OS. This allows to notify the guest operating system"
 682             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 683             ." execution or creation from a template). The guest operating system notices the"
 684             ." change, and is then able to react as appropriate by marking its copies of"
 685             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 686             ."Note that auto-creation only works when done through API/CLI create or update methods"
 687             .", but not when manually editing the config file.",
 688         default => "1 (autogenerated)",
 689         optional => 1,
 690     },
 691     hookscript => {
 692         type => 'string',
 693         format => 'pve-volume-id',
 694         optional => 1,
 695         description => "Script that will be executed during various steps in the vms lifetime.",
 696     },
 697     ivshmem => {
 698         type => 'string',
 699         format => $ivshmem_fmt,
 700         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 701             ." the host.",
 702         optional => 1,
 703     },
 704     audio0 => {
 705         type => 'string',
 706         format => $audio_fmt,
 707         description => "Configure a audio device, useful in combination with QXL/Spice.",
 708         optional => 1
 709     },
 710     spice_enhancements => {
 711         type => 'string',
 712         format => $spice_enhancements_fmt,
 713         description => "Configure additional enhancements for SPICE.",
 714         optional => 1
 715     },
 716     tags => {
 717         type => 'string', format => 'pve-tag-list',
 718         description => 'Tags of the VM. This is only meta information.',
 719         optional => 1,
 720     },
 721     rng0 => {
 722         type => 'string',
 723         format => $rng_fmt,
 724         description => "Configure a VirtIO-based Random Number Generator.",
 725         optional => 1,
 726     },
 727     meta => {
 728         type => 'string',
 729         format => $meta_info_fmt,
 730         description => "Some (read-only) meta-information about this guest.",
 731         optional => 1,
 732     },
 733     affinity => {
 734         type => 'string', format => 'pve-cpuset',
 735         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 736         optional => 1,
 737     },
 738 };
 739
 740 my $cicustom_fmt = {
 741     meta => {
 742         type => 'string',
 743         optional => 1,
 744         description => 'Specify a custom file containing all meta data passed to the VM via"
 745             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 746         format => 'pve-volume-id',
 747         format_description => 'volume',
 748     },
 749     network => {
 750         type => 'string',
 751         optional => 1,
 752         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 753         format => 'pve-volume-id',
 754         format_description => 'volume',
 755     },
 756     user => {
 757         type => 'string',
 758         optional => 1,
 759         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 760         format => 'pve-volume-id',
 761         format_description => 'volume',
 762     },
 763     vendor => {
 764         type => 'string',
 765         optional => 1,
 766         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 767         format => 'pve-volume-id',
 768         format_description => 'volume',
 769     },
 770 };
 771 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 772
 773 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 774 my $confdesc_cloudinit = {
 775     citype => {
 776         optional => 1,
 777         type => 'string',
 778         description => 'Specifies the cloud-init configuration format. The default depends on the'
 779             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 780             .' and `configdrive2` for windows.',
 781         enum => ['configdrive2', 'nocloud', 'opennebula'],
 782     },
 783     ciuser => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: User name to change ssh keys and password for instead of the"
 787             ." image's configured default user.",
 788     },
 789     cipassword => {
 790         optional => 1,
 791         type => 'string',
 792         description => 'cloud-init: Password to assign the user. Using this is generally not'
 793             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 794             .' support hashed passwords.',
 795     },
 796     ciupgrade => {
 797         optional => 1,
 798         type => 'boolean',
 799         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 800         default => 1,
 801     },
 802     cicustom => {
 803         optional => 1,
 804         type => 'string',
 805         description => 'cloud-init: Specify custom files to replace the automatically generated'
 806             .' ones at start.',
 807         format => 'pve-qm-cicustom',
 808     },
 809     searchdomain => {
 810         optional => 1,
 811         type => 'string',
 812         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 813             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 814             .' are set.',
 815     },
 816     nameserver => {
 817         optional => 1,
 818         type => 'string', format => 'address-list',
 819         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 820             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 821             .' are set.',
 822     },
 823     sshkeys => {
 824         optional => 1,
 825         type => 'string',
 826         format => 'urlencoded',
 827         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 828     },
 829 };
 830
 831 # what about other qemu settings ?
 832 #cpu => 'string',
 833 #machine => 'string',
 834 #fda => 'file',
 835 #fdb => 'file',
 836 #mtdblock => 'file',
 837 #sd => 'file',
 838 #pflash => 'file',
 839 #snapshot => 'bool',
 840 #bootp => 'file',
 841 ##tftp => 'dir',
 842 ##smb => 'dir',
 843 #kernel => 'file',
 844 #append => 'string',
 845 #initrd => 'file',
 846 ##soundhw => 'string',
 847
 848 while (my ($k, $v) = each %$confdesc) {
 849     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 850 }
 851
 852 my $MAX_NETS = 32;
 853 my $MAX_SERIAL_PORTS = 4;
 854 my $MAX_PARALLEL_PORTS = 3;
 855
 856 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 857     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 858 }
 859
 860 my $nic_model_list = [
 861     'e1000',
 862     'e1000-82540em',
 863     'e1000-82544gc',
 864     'e1000-82545em',
 865     'e1000e',
 866     'i82551',
 867     'i82557b',
 868     'i82559er',
 869     'ne2k_isa',
 870     'ne2k_pci',
 871     'pcnet',
 872     'rtl8139',
 873     'virtio',
 874     'vmxnet3',
 875 ];
 876 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 877
 878 my $net_fmt_bridge_descr = <<__EOD__;
 879 Bridge to attach the network device to. The Proxmox VE standard bridge
 880 is called 'vmbr0'.
 881
 882 If you do not specify a bridge, we create a kvm user (NATed) network
 883 device, which provides DHCP and DNS services. The following addresses
 884 are used:
 885
 886  10.0.2.2   Gateway
 887  10.0.2.3   DNS Server
 888  10.0.2.4   SMB Server
 889
 890 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 891 __EOD__
 892
 893 my $net_fmt = {
 894     macaddr  => get_standard_option('mac-addr', {
 895         description => "MAC address. That address must be unique withing your network. This is"
 896             ." automatically generated if not specified.",
 897     }),
 898     model => {
 899         type => 'string',
 900         description => "Network Card Model. The 'virtio' model provides the best performance with"
 901             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 902             ." best to use 'e1000'.",
 903         enum => $nic_model_list,
 904         default_key => 1,
 905     },
 906     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 907     bridge => get_standard_option('pve-bridge-id', {
 908         description => $net_fmt_bridge_descr,
 909         optional => 1,
 910     }),
 911     queues => {
 912         type => 'integer',
 913         minimum => 0, maximum => 64,
 914         description => 'Number of packet queues to be used on the device.',
 915         optional => 1,
 916     },
 917     rate => {
 918         type => 'number',
 919         minimum => 0,
 920         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 921         optional => 1,
 922     },
 923     tag => {
 924         type => 'integer',
 925         minimum => 1, maximum => 4094,
 926         description => 'VLAN tag to apply to packets on this interface.',
 927         optional => 1,
 928     },
 929     trunks => {
 930         type => 'string',
 931         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 932         description => 'VLAN trunks to pass through this interface.',
 933         format_description => 'vlanid[;vlanid...]',
 934         optional => 1,
 935     },
 936     firewall => {
 937         type => 'boolean',
 938         description => 'Whether this interface should be protected by the firewall.',
 939         optional => 1,
 940     },
 941     link_down => {
 942         type => 'boolean',
 943         description => 'Whether this interface should be disconnected (like pulling the plug).',
 944         optional => 1,
 945     },
 946     mtu => {
 947         type => 'integer',
 948         minimum => 1, maximum => 65520,
 949         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 950         optional => 1,
 951     },
 952 };
 953
 954 my $netdesc = {
 955     optional => 1,
 956     type => 'string', format => $net_fmt,
 957     description => "Specify network devices.",
 958 };
 959
 960 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 961
 962 my $ipconfig_fmt = {
 963     ip => {
 964         type => 'string',
 965         format => 'pve-ipv4-config',
 966         format_description => 'IPv4Format/CIDR',
 967         description => 'IPv4 address in CIDR format.',
 968         optional => 1,
 969         default => 'dhcp',
 970     },
 971     gw => {
 972         type => 'string',
 973         format => 'ipv4',
 974         format_description => 'GatewayIPv4',
 975         description => 'Default gateway for IPv4 traffic.',
 976         optional => 1,
 977         requires => 'ip',
 978     },
 979     ip6 => {
 980         type => 'string',
 981         format => 'pve-ipv6-config',
 982         format_description => 'IPv6Format/CIDR',
 983         description => 'IPv6 address in CIDR format.',
 984         optional => 1,
 985         default => 'dhcp',
 986     },
 987     gw6 => {
 988         type => 'string',
 989         format => 'ipv6',
 990         format_description => 'GatewayIPv6',
 991         description => 'Default gateway for IPv6 traffic.',
 992         optional => 1,
 993         requires => 'ip6',
 994     },
 995 };
 996 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 997 my $ipconfigdesc = {
 998     optional => 1,
 999     type => 'string', format => 'pve-qm-ipconfig',
1000     description => <<'EODESCR',
1001 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1002
1003 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1004
1005 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1006 gateway should be provided.
1007 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1008 cloud-init 19.4 or newer.
1009
1010 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1011 dhcp on IPv4.
1012 EODESCR
1013 };
1014 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1015
1016 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1017     $confdesc->{"net$i"} = $netdesc;
1018     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1019 }
1020
1021 foreach my $key (keys %$confdesc_cloudinit) {
1022     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1023 }
1024
1025 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1026 sub pve_verify_cpuset {
1027     my ($set_text, $noerr) = @_;
1028
1029     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1030
1031     if ($@) {
1032         return if $noerr;
1033         die "unable to parse cpuset option\n";
1034     }
1035
1036     return PVE::CpuSet->new($members)->short_string();
1037 }
1038
1039 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1040 sub verify_volume_id_or_qm_path {
1041     my ($volid, $noerr) = @_;
1042
1043     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1044
1045     return verify_volume_id_or_absolute_path($volid, $noerr);
1046 }
1047
1048 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1049 sub verify_volume_id_or_absolute_path {
1050     my ($volid, $noerr) = @_;
1051
1052     return $volid if $volid =~ m|^/|;
1053
1054     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1055     if ($@) {
1056         return if $noerr;
1057         die $@;
1058     }
1059     return $volid;
1060 }
1061
1062 my $serialdesc = {
1063         optional => 1,
1064         type => 'string',
1065         pattern => '(/dev/.+|socket)',
1066         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1067         verbose_description =>  <<EODESCR,
1068 Create a serial device inside the VM (n is 0 to 3), and pass through a
1069 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1070 host side (use 'qm terminal' to open a terminal connection).
1071
1072 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1073 use with special care.
1074
1075 CAUTION: Experimental! User reported problems with this option.
1076 EODESCR
1077 };
1078
1079 my $paralleldesc= {
1080         optional => 1,
1081         type => 'string',
1082         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1083         description =>  "Map host parallel devices (n is 0 to 2).",
1084         verbose_description =>  <<EODESCR,
1085 Map host parallel devices (n is 0 to 2).
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1095     $confdesc->{"parallel$i"} = $paralleldesc;
1096 }
1097
1098 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1099     $confdesc->{"serial$i"} = $serialdesc;
1100 }
1101
1102 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1103     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1104 }
1105
1106 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1107     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1108 }
1109
1110 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1111     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1112 }
1113
1114 my $boot_fmt = {
1115     legacy => {
1116         optional => 1,
1117         default_key => 1,
1118         type => 'string',
1119         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1120                      . " Deprecated, use 'order=' instead.",
1121         pattern => '[acdn]{1,4}',
1122         format_description => "[acdn]{1,4}",
1123
1124         # note: this is also the fallback if boot: is not given at all
1125         default => 'cdn',
1126     },
1127     order => {
1128         optional => 1,
1129         type => 'string',
1130         format => 'pve-qm-bootdev-list',
1131         format_description => "device[;device...]",
1132         description => <<EODESC,
1133 The guest will attempt to boot from devices in the order they appear here.
1134
1135 Disks, optical drives and passed-through storage USB devices will be directly
1136 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1137 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1138
1139 Note that only devices in this list will be marked as bootable and thus loaded
1140 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1141 (e.g. software-raid), you need to specify all of them here.
1142
1143 Overrides the deprecated 'legacy=[acdn]*' value when given.
1144 EODESC
1145     },
1146 };
1147 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1148
1149 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1150 sub verify_bootdev {
1151     my ($dev, $noerr) = @_;
1152
1153     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1154     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1155
1156     my $check = sub {
1157         my ($base) = @_;
1158         return 0 if $dev !~ m/^$base\d+$/;
1159         return 0 if !$confdesc->{$dev};
1160         return 1;
1161     };
1162
1163     return $dev if $check->("net");
1164     return $dev if $check->("usb");
1165     return $dev if $check->("hostpci");
1166
1167     return if $noerr;
1168     die "invalid boot device '$dev'\n";
1169 }
1170
1171 sub print_bootorder {
1172     my ($devs) = @_;
1173     return "" if !@$devs;
1174     my $data = { order => join(';', @$devs) };
1175     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1176 }
1177
1178 my $kvm_api_version = 0;
1179
1180 sub kvm_version {
1181     return $kvm_api_version if $kvm_api_version;
1182
1183     open my $fh, '<', '/dev/kvm' or return;
1184
1185     # 0xae00 => KVM_GET_API_VERSION
1186     $kvm_api_version = ioctl($fh, 0xae00, 0);
1187     close($fh);
1188
1189     return $kvm_api_version;
1190 }
1191
1192 my $kvm_user_version = {};
1193 my $kvm_mtime = {};
1194
1195 sub kvm_user_version {
1196     my ($binary) = @_;
1197
1198     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1199     my $st = stat($binary);
1200
1201     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1202     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1203         $cachedmtime == $st->mtime;
1204
1205     $kvm_user_version->{$binary} = 'unknown';
1206     $kvm_mtime->{$binary} = $st->mtime;
1207
1208     my $code = sub {
1209         my $line = shift;
1210         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1211             $kvm_user_version->{$binary} = $2;
1212         }
1213     };
1214
1215     eval { run_command([$binary, '--version'], outfunc => $code); };
1216     warn $@ if $@;
1217
1218     return $kvm_user_version->{$binary};
1219
1220 }
1221 my sub extract_version {
1222     my ($machine_type, $version) = @_;
1223     $version = kvm_user_version() if !defined($version);
1224     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1225 }
1226
1227 sub kernel_has_vhost_net {
1228     return -c '/dev/vhost-net';
1229 }
1230
1231 sub option_exists {
1232     my $key = shift;
1233     return defined($confdesc->{$key});
1234 }
1235
1236 my $cdrom_path;
1237 sub get_cdrom_path {
1238
1239     return $cdrom_path if defined($cdrom_path);
1240
1241     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1242
1243     if (!defined($cdrom_path)) {
1244         log_warn("no physical CD-ROM available, ignoring");
1245         $cdrom_path = '';
1246     }
1247
1248     return $cdrom_path;
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269      if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1314             if !$vtype;
1315         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1316         verify_media_type($opt, $vtype, $drive->{media});
1317         $drive->{file} = $volid;
1318     }
1319
1320     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1321 }
1322
1323 sub parse_hotplug_features {
1324     my ($data) = @_;
1325
1326     my $res = {};
1327
1328     return $res if $data eq '0';
1329
1330     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1331
1332     foreach my $feature (PVE::Tools::split_list($data)) {
1333         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1334             $res->{$1} = 1;
1335         } else {
1336             die "invalid hotplug feature '$feature'\n";
1337         }
1338     }
1339     return $res;
1340 }
1341
1342 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1343 sub pve_verify_hotplug_features {
1344     my ($value, $noerr) = @_;
1345
1346     return $value if parse_hotplug_features($value);
1347
1348     return if $noerr;
1349
1350     die "unable to parse hotplug option\n";
1351 }
1352
1353 sub assert_clipboard_config {
1354     my ($vga) = @_;
1355
1356     my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1357
1358     if (
1359         $vga->{'clipboard'}
1360         && $vga->{'clipboard'} eq 'vnc'
1361         && $vga->{type}
1362         && $vga->{type} !~ $clipboard_regex
1363     ) {
1364         die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1365     }
1366 }
1367
1368 sub print_tabletdevice_full {
1369     my ($conf, $arch) = @_;
1370
1371     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1372
1373     # we use uhci for old VMs because tablet driver was buggy in older qemu
1374     my $usbbus;
1375     if ($q35 || $arch eq 'aarch64') {
1376         $usbbus = 'ehci';
1377     } else {
1378         $usbbus = 'uhci';
1379     }
1380
1381     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1382 }
1383
1384 sub print_keyboarddevice_full {
1385     my ($conf, $arch) = @_;
1386
1387     return if $arch ne 'aarch64';
1388
1389     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1390 }
1391
1392 my sub get_drive_id {
1393     my ($drive) = @_;
1394     return "$drive->{interface}$drive->{index}";
1395 }
1396
1397 sub print_drivedevice_full {
1398     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1399
1400     my $device = '';
1401     my $maxdev = 0;
1402
1403     my $drive_id = get_drive_id($drive);
1404     if ($drive->{interface} eq 'virtio') {
1405         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1406         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1407         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1408     } elsif ($drive->{interface} eq 'scsi') {
1409
1410         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1411         my $unit = $drive->{index} % $maxdev;
1412
1413         my $machine_version = extract_version($machine_type, kvm_user_version());
1414         my $devicetype  = PVE::QemuServer::Drive::get_scsi_devicetype(
1415             $drive, $storecfg, $machine_version);
1416
1417         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1418             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1419         } else {
1420             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1421                 .",lun=$drive->{index}";
1422         }
1423         $device .= ",drive=drive-$drive_id,id=$drive_id";
1424
1425         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1426             $device .= ",rotation_rate=1";
1427         }
1428         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1429
1430         # only scsi-hd and scsi-cd support passing vendor and product information
1431         if ($devicetype eq 'hd' || $devicetype eq 'cd') {
1432             if (my $vendor = $drive->{vendor}) {
1433                 $device .= ",vendor=$vendor";
1434             }
1435             if (my $product = $drive->{product}) {
1436                 $device .= ",product=$product";
1437             }
1438         }
1439
1440     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1441         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1442         my $controller = int($drive->{index} / $maxdev);
1443         my $unit = $drive->{index} % $maxdev;
1444
1445         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1446         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1447         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1448         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1449         # were before. Move odd ones up by 2 where they don't clash.
1450         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1451             $controller += 2 * ($unit % 2);
1452             $unit = 0;
1453         }
1454
1455         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1456
1457         $device = "ide-$devicetype";
1458         if ($drive->{interface} eq 'ide') {
1459             $device .= ",bus=ide.$controller,unit=$unit";
1460         } else {
1461             $device .= ",bus=ahci$controller.$unit";
1462         }
1463         $device .= ",drive=drive-$drive_id,id=$drive_id";
1464
1465         if ($devicetype eq 'hd') {
1466             if (my $model = $drive->{model}) {
1467                 $model = URI::Escape::uri_unescape($model);
1468                 $device .= ",model=$model";
1469             }
1470             if ($drive->{ssd}) {
1471                 $device .= ",rotation_rate=1";
1472             }
1473         }
1474         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1475     } elsif ($drive->{interface} eq 'usb') {
1476         die "implement me";
1477         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1478     } else {
1479         die "unsupported interface type";
1480     }
1481
1482     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1483
1484     if (my $serial = $drive->{serial}) {
1485         $serial = URI::Escape::uri_unescape($serial);
1486         $device .= ",serial=$serial";
1487     }
1488
1489
1490     return $device;
1491 }
1492
1493 sub get_initiator_name {
1494     my $initiator;
1495
1496     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1497     while (defined(my $line = <$fh>)) {
1498         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1499         $initiator = $1;
1500         last;
1501     }
1502     $fh->close();
1503
1504     return $initiator;
1505 }
1506
1507 my sub storage_allows_io_uring_default {
1508     my ($scfg, $cache_direct) = @_;
1509
1510     # io_uring with cache mode writeback or writethrough on krbd will hang...
1511     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1512
1513     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1514     # sometimes, just plain disable...
1515     return if $scfg && $scfg->{type} eq 'lvm';
1516
1517     # io_uring causes problems when used with CIFS since kernel 5.15
1518     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1519     return if $scfg && $scfg->{type} eq 'cifs';
1520
1521     return 1;
1522 }
1523
1524 my sub drive_uses_cache_direct {
1525     my ($drive, $scfg) = @_;
1526
1527     my $cache_direct = 0;
1528
1529     if (my $cache = $drive->{cache}) {
1530         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1531     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1532         $cache_direct = 1;
1533     }
1534
1535     return $cache_direct;
1536 }
1537
1538 sub print_drive_commandline_full {
1539     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1540
1541     my $path;
1542     my $volid = $drive->{file};
1543     my $format = $drive->{format};
1544     my $drive_id = get_drive_id($drive);
1545
1546     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1547     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1548
1549     if (drive_is_cdrom($drive)) {
1550         $path = get_iso_path($storecfg, $vmid, $volid);
1551         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1552     } else {
1553         if ($storeid) {
1554             $path = PVE::Storage::path($storecfg, $volid);
1555             $format //= qemu_img_format($scfg, $volname);
1556         } else {
1557             $path = $volid;
1558             $format //= "raw";
1559         }
1560    }
1561
1562    my $is_rbd = $path =~ m/^rbd:/;
1563
1564     my $opts = '';
1565     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1566     foreach my $o (@qemu_drive_options) {
1567         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1568     }
1569
1570     # snapshot only accepts on|off
1571     if (defined($drive->{snapshot})) {
1572         my $v = $drive->{snapshot} ? 'on' : 'off';
1573         $opts .= ",snapshot=$v";
1574     }
1575
1576     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1577         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1578     }
1579
1580     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1581         my ($dir, $qmpname) = @$type;
1582         if (my $v = $drive->{"mbps$dir"}) {
1583             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1584         }
1585         if (my $v = $drive->{"mbps${dir}_max"}) {
1586             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1587         }
1588         if (my $v = $drive->{"bps${dir}_max_length"}) {
1589             $opts .= ",throttling.bps$qmpname-max-length=$v";
1590         }
1591         if (my $v = $drive->{"iops${dir}"}) {
1592             $opts .= ",throttling.iops$qmpname=$v";
1593         }
1594         if (my $v = $drive->{"iops${dir}_max"}) {
1595             $opts .= ",throttling.iops$qmpname-max=$v";
1596         }
1597         if (my $v = $drive->{"iops${dir}_max_length"}) {
1598             $opts .= ",throttling.iops$qmpname-max-length=$v";
1599         }
1600     }
1601
1602     if ($pbs_name) {
1603         $format = "rbd" if $is_rbd;
1604         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1605             if !$format;
1606         $opts .= ",format=alloc-track,file.driver=$format";
1607     } elsif ($format) {
1608         $opts .= ",format=$format";
1609     }
1610
1611     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1612
1613     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1614
1615     if (!$drive->{aio}) {
1616         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1617             # io_uring supports all cache modes
1618             $opts .= ",aio=io_uring";
1619         } else {
1620             # aio native works only with O_DIRECT
1621             if($cache_direct) {
1622                 $opts .= ",aio=native";
1623             } else {
1624                 $opts .= ",aio=threads";
1625             }
1626         }
1627     }
1628
1629     if (!drive_is_cdrom($drive)) {
1630         my $detectzeroes;
1631         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1632             $detectzeroes = 'off';
1633         } elsif ($drive->{discard}) {
1634             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1635         } else {
1636             # This used to be our default with discard not being specified:
1637             $detectzeroes = 'on';
1638         }
1639
1640         # note: 'detect-zeroes' works per blockdev and we want it to persist
1641         # after the alloc-track is removed, so put it on 'file' directly
1642         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1643         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1644     }
1645
1646     if ($pbs_name) {
1647         $opts .= ",backing=$pbs_name";
1648         $opts .= ",auto-remove=on";
1649     }
1650
1651     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1652     my $file_param = "file";
1653     if ($pbs_name) {
1654         # non-rbd drivers require the underlying file to be a seperate block
1655         # node, so add a second .file indirection
1656         $file_param .= ".file" if !$is_rbd;
1657         $file_param .= ".filename";
1658     }
1659     my $pathinfo = $path ? "$file_param=$path," : '';
1660
1661     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1662 }
1663
1664 sub print_pbs_blockdev {
1665     my ($pbs_conf, $pbs_name) = @_;
1666     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1667     $blockdev .= ",repository=$pbs_conf->{repository}";
1668     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1669     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1670     $blockdev .= ",archive=$pbs_conf->{archive}";
1671     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1672     return $blockdev;
1673 }
1674
1675 sub print_netdevice_full {
1676     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1677
1678     my $device = $net->{model};
1679     if ($net->{model} eq 'virtio') {
1680          $device = 'virtio-net-pci';
1681      };
1682
1683     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1684     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1685     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1686         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1687         # and out of each queue plus one config interrupt and control vector queue
1688         my $vectors = $net->{queues} * 2 + 2;
1689         $tmpstr .= ",vectors=$vectors,mq=on";
1690         if (min_version($machine_version, 7, 1)) {
1691             $tmpstr .= ",packed=on";
1692         }
1693     }
1694
1695     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1696         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1697     }
1698
1699     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1700
1701     if (my $mtu = $net->{mtu}) {
1702         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1703             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1704             if ($mtu == 1) {
1705                  $mtu = $bridge_mtu;
1706             } elsif ($mtu < 576) {
1707                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1708             } elsif ($mtu > $bridge_mtu) {
1709                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1710             }
1711             $tmpstr .= ",host_mtu=$mtu";
1712         } else {
1713             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1714         }
1715     }
1716
1717     if ($use_old_bios_files) {
1718         my $romfile;
1719         if ($device eq 'virtio-net-pci') {
1720             $romfile = 'pxe-virtio.rom';
1721         } elsif ($device eq 'e1000') {
1722             $romfile = 'pxe-e1000.rom';
1723         } elsif ($device eq 'e1000e') {
1724             $romfile = 'pxe-e1000e.rom';
1725         } elsif ($device eq 'ne2k') {
1726             $romfile = 'pxe-ne2k_pci.rom';
1727         } elsif ($device eq 'pcnet') {
1728             $romfile = 'pxe-pcnet.rom';
1729         } elsif ($device eq 'rtl8139') {
1730             $romfile = 'pxe-rtl8139.rom';
1731         }
1732         $tmpstr .= ",romfile=$romfile" if $romfile;
1733     }
1734
1735     return $tmpstr;
1736 }
1737
1738 sub print_netdev_full {
1739     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1740
1741     my $i = '';
1742     if ($netid =~ m/^net(\d+)$/) {
1743         $i = int($1);
1744     }
1745
1746     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1747
1748     my $ifname = "tap${vmid}i$i";
1749
1750     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1751     die "interface name '$ifname' is too long (max 15 character)\n"
1752         if length($ifname) >= 16;
1753
1754     my $vhostparam = '';
1755     if (is_native($arch)) {
1756         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1757     }
1758
1759     my $vmname = $conf->{name} || "vm$vmid";
1760
1761     my $netdev = "";
1762     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1763
1764     if ($net->{bridge}) {
1765         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1766             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1767     } else {
1768         $netdev = "type=user,id=$netid,hostname=$vmname";
1769     }
1770
1771     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1772
1773     return $netdev;
1774 }
1775
1776 my $vga_map = {
1777     'cirrus' => 'cirrus-vga',
1778     'std' => 'VGA',
1779     'vmware' => 'vmware-svga',
1780     'virtio' => 'virtio-vga',
1781     'virtio-gl' => 'virtio-vga-gl',
1782 };
1783
1784 sub print_vga_device {
1785     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1786
1787     my $type = $vga_map->{$vga->{type}};
1788     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1789         $type = 'virtio-gpu';
1790     }
1791     my $vgamem_mb = $vga->{memory};
1792
1793     my $max_outputs = '';
1794     if ($qxlnum) {
1795         $type = $id ? 'qxl' : 'qxl-vga';
1796
1797         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1798             # set max outputs so linux can have up to 4 qxl displays with one device
1799             if (min_version($machine_version, 4, 1)) {
1800                 $max_outputs = ",max_outputs=4";
1801             }
1802         }
1803     }
1804
1805     die "no devicetype for $vga->{type}\n" if !$type;
1806
1807     my $memory = "";
1808     if ($vgamem_mb) {
1809         if ($vga->{type} =~ /^virtio/) {
1810             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1811             $memory = ",max_hostmem=$bytes";
1812         } elsif ($qxlnum) {
1813             # from https://www.spice-space.org/multiple-monitors.html
1814             $memory = ",vgamem_mb=$vga->{memory}";
1815             my $ram = $vgamem_mb * 4;
1816             my $vram = $vgamem_mb * 2;
1817             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1818         } else {
1819             $memory = ",vgamem_mb=$vga->{memory}";
1820         }
1821     } elsif ($qxlnum && $id) {
1822         $memory = ",ram_size=67108864,vram_size=33554432";
1823     }
1824
1825     my $edidoff = "";
1826     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1827         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1828     }
1829
1830     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1831     my $vgaid = "vga" . ($id // '');
1832     my $pciaddr;
1833     if ($q35 && $vgaid eq 'vga') {
1834         # the first display uses pcie.0 bus on q35 machines
1835         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1836     } else {
1837         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1838     }
1839
1840     if ($vga->{type} eq 'virtio-gl') {
1841         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1842         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1843             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1844
1845         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1846             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1847     }
1848
1849     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1850 }
1851
1852 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1853 sub parse_net {
1854     my ($data, $disable_mac_autogen) = @_;
1855
1856     my $res = eval { parse_property_string($net_fmt, $data) };
1857     if ($@) {
1858         warn $@;
1859         return;
1860     }
1861     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1862         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1863         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1864     }
1865     return $res;
1866 }
1867
1868 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1869 sub parse_ipconfig {
1870     my ($data) = @_;
1871
1872     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1873     if ($@) {
1874         warn $@;
1875         return;
1876     }
1877
1878     if ($res->{gw} && !$res->{ip}) {
1879         warn 'gateway specified without specifying an IP address';
1880         return;
1881     }
1882     if ($res->{gw6} && !$res->{ip6}) {
1883         warn 'IPv6 gateway specified without specifying an IPv6 address';
1884         return;
1885     }
1886     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1887         warn 'gateway specified together with DHCP';
1888         return;
1889     }
1890     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1891         # gw6 + auto/dhcp
1892         warn "IPv6 gateway specified together with $res->{ip6} address";
1893         return;
1894     }
1895
1896     if (!$res->{ip} && !$res->{ip6}) {
1897         return { ip => 'dhcp', ip6 => 'dhcp' };
1898     }
1899
1900     return $res;
1901 }
1902
1903 sub print_net {
1904     my $net = shift;
1905
1906     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1907 }
1908
1909 sub add_random_macs {
1910     my ($settings) = @_;
1911
1912     foreach my $opt (keys %$settings) {
1913         next if $opt !~ m/^net(\d+)$/;
1914         my $net = parse_net($settings->{$opt});
1915         next if !$net;
1916         $settings->{$opt} = print_net($net);
1917     }
1918 }
1919
1920 sub vm_is_volid_owner {
1921     my ($storecfg, $vmid, $volid) = @_;
1922
1923     if ($volid !~  m|^/|) {
1924         my ($path, $owner);
1925         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1926         if ($owner && ($owner == $vmid)) {
1927             return 1;
1928         }
1929     }
1930
1931     return;
1932 }
1933
1934 sub vmconfig_register_unused_drive {
1935     my ($storecfg, $vmid, $conf, $drive) = @_;
1936
1937     if (drive_is_cloudinit($drive)) {
1938         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1939         warn $@ if $@;
1940         delete $conf->{cloudinit};
1941     } elsif (!drive_is_cdrom($drive)) {
1942         my $volid = $drive->{file};
1943         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1944             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1945         }
1946     }
1947 }
1948
1949 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1950 my $smbios1_fmt = {
1951     uuid => {
1952         type => 'string',
1953         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1954         format_description => 'UUID',
1955         description => "Set SMBIOS1 UUID.",
1956         optional => 1,
1957     },
1958     version => {
1959         type => 'string',
1960         pattern => '[A-Za-z0-9+\/]+={0,2}',
1961         format_description => 'Base64 encoded string',
1962         description => "Set SMBIOS1 version.",
1963         optional => 1,
1964     },
1965     serial => {
1966         type => 'string',
1967         pattern => '[A-Za-z0-9+\/]+={0,2}',
1968         format_description => 'Base64 encoded string',
1969         description => "Set SMBIOS1 serial number.",
1970         optional => 1,
1971     },
1972     manufacturer => {
1973         type => 'string',
1974         pattern => '[A-Za-z0-9+\/]+={0,2}',
1975         format_description => 'Base64 encoded string',
1976         description => "Set SMBIOS1 manufacturer.",
1977         optional => 1,
1978     },
1979     product => {
1980         type => 'string',
1981         pattern => '[A-Za-z0-9+\/]+={0,2}',
1982         format_description => 'Base64 encoded string',
1983         description => "Set SMBIOS1 product ID.",
1984         optional => 1,
1985     },
1986     sku => {
1987         type => 'string',
1988         pattern => '[A-Za-z0-9+\/]+={0,2}',
1989         format_description => 'Base64 encoded string',
1990         description => "Set SMBIOS1 SKU string.",
1991         optional => 1,
1992     },
1993     family => {
1994         type => 'string',
1995         pattern => '[A-Za-z0-9+\/]+={0,2}',
1996         format_description => 'Base64 encoded string',
1997         description => "Set SMBIOS1 family string.",
1998         optional => 1,
1999     },
2000     base64 => {
2001         type => 'boolean',
2002         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2003         optional => 1,
2004     },
2005 };
2006
2007 sub parse_smbios1 {
2008     my ($data) = @_;
2009
2010     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2011     warn $@ if $@;
2012     return $res;
2013 }
2014
2015 sub print_smbios1 {
2016     my ($smbios1) = @_;
2017     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2018 }
2019
2020 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2021
2022 sub parse_watchdog {
2023     my ($value) = @_;
2024
2025     return if !$value;
2026
2027     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2028     warn $@ if $@;
2029     return $res;
2030 }
2031
2032 sub parse_guest_agent {
2033     my ($conf) = @_;
2034
2035     return {} if !defined($conf->{agent});
2036
2037     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2038     warn $@ if $@;
2039
2040     # if the agent is disabled ignore the other potentially set properties
2041     return {} if !$res->{enabled};
2042     return $res;
2043 }
2044
2045 sub get_qga_key {
2046     my ($conf, $key) = @_;
2047     return undef if !defined($conf->{agent});
2048
2049     my $agent = parse_guest_agent($conf);
2050     return $agent->{$key};
2051 }
2052
2053 sub parse_vga {
2054     my ($value) = @_;
2055
2056     return {} if !$value;
2057     my $res = eval { parse_property_string($vga_fmt, $value) };
2058     warn $@ if $@;
2059     return $res;
2060 }
2061
2062 sub parse_rng {
2063     my ($value) = @_;
2064
2065     return if !$value;
2066
2067     my $res = eval { parse_property_string($rng_fmt, $value) };
2068     warn $@ if $@;
2069     return $res;
2070 }
2071
2072 sub parse_meta_info {
2073     my ($value) = @_;
2074
2075     return if !$value;
2076
2077     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2078     warn $@ if $@;
2079     return $res;
2080 }
2081
2082 sub new_meta_info_string {
2083     my () = @_; # for now do not allow to override any value
2084
2085     return PVE::JSONSchema::print_property_string(
2086         {
2087             'creation-qemu' => kvm_user_version(),
2088             ctime => "". int(time()),
2089         },
2090         $meta_info_fmt
2091     );
2092 }
2093
2094 sub qemu_created_version_fixups {
2095     my ($conf, $forcemachine, $kvmver) = @_;
2096
2097     my $meta = parse_meta_info($conf->{meta}) // {};
2098     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2099
2100     # check if we need to apply some handling for VMs that always use the latest machine version but
2101     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2102     # would be required (we do not want to pin machine version for non-windows OS type)
2103     if (
2104         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2105         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2106         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2107         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2108     ) {
2109         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2110         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2111             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2112             # and thus with the predictable interface naming of systemd
2113             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2114         }
2115     }
2116     return;
2117 }
2118
2119 # add JSON properties for create and set function
2120 sub json_config_properties {
2121     my ($prop, $with_disk_alloc) = @_;
2122
2123     my $skip_json_config_opts = {
2124         parent => 1,
2125         snaptime => 1,
2126         vmstate => 1,
2127         runningmachine => 1,
2128         runningcpu => 1,
2129         meta => 1,
2130     };
2131
2132     foreach my $opt (keys %$confdesc) {
2133         next if $skip_json_config_opts->{$opt};
2134
2135         if ($with_disk_alloc && is_valid_drivename($opt)) {
2136             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2137         } else {
2138             $prop->{$opt} = $confdesc->{$opt};
2139         }
2140     }
2141
2142     return $prop;
2143 }
2144
2145 # Properties that we can read from an OVF file
2146 sub json_ovf_properties {
2147     my $prop = {};
2148
2149     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2150         $prop->{$device} = {
2151             type => 'string',
2152             format => 'pve-volume-id-or-absolute-path',
2153             description => "Disk image that gets imported to $device",
2154             optional => 1,
2155         };
2156     }
2157
2158     $prop->{cores} = {
2159         type => 'integer',
2160         description => "The number of CPU cores.",
2161         optional => 1,
2162     };
2163     $prop->{memory} = {
2164         type => 'integer',
2165         description => "Amount of RAM for the VM in MB.",
2166         optional => 1,
2167     };
2168     $prop->{name} = {
2169         type => 'string',
2170         description => "Name of the VM.",
2171         optional => 1,
2172     };
2173
2174     return $prop;
2175 }
2176
2177 # return copy of $confdesc_cloudinit to generate documentation
2178 sub cloudinit_config_properties {
2179
2180     return dclone($confdesc_cloudinit);
2181 }
2182
2183 sub cloudinit_pending_properties {
2184     my $p = {
2185         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2186         name => 1,
2187     };
2188     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2189     return $p;
2190 }
2191
2192 sub check_type {
2193     my ($key, $value) = @_;
2194
2195     die "unknown setting '$key'\n" if !$confdesc->{$key};
2196
2197     my $type = $confdesc->{$key}->{type};
2198
2199     if (!defined($value)) {
2200         die "got undefined value\n";
2201     }
2202
2203     if ($value =~ m/[\n\r]/) {
2204         die "property contains a line feed\n";
2205     }
2206
2207     if ($type eq 'boolean') {
2208         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2209         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2210         die "type check ('boolean') failed - got '$value'\n";
2211     } elsif ($type eq 'integer') {
2212         return int($1) if $value =~ m/^(\d+)$/;
2213         die "type check ('integer') failed - got '$value'\n";
2214     } elsif ($type eq 'number') {
2215         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2216         die "type check ('number') failed - got '$value'\n";
2217     } elsif ($type eq 'string') {
2218         if (my $fmt = $confdesc->{$key}->{format}) {
2219             PVE::JSONSchema::check_format($fmt, $value);
2220             return $value;
2221         }
2222         $value =~ s/^\"(.*)\"$/$1/;
2223         return $value;
2224     } else {
2225         die "internal error"
2226     }
2227 }
2228
2229 sub destroy_vm {
2230     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2231
2232     my $conf = PVE::QemuConfig->load_config($vmid);
2233
2234     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2235         PVE::QemuConfig->check_lock($conf);
2236     }
2237
2238     if ($conf->{template}) {
2239         # check if any base image is still used by a linked clone
2240         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2241                 my ($ds, $drive) = @_;
2242                 return if drive_is_cdrom($drive);
2243
2244                 my $volid = $drive->{file};
2245                 return if !$volid || $volid =~ m|^/|;
2246
2247                 die "base volume '$volid' is still in use by linked cloned\n"
2248                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2249
2250         });
2251     }
2252
2253     my $volids = {};
2254     my $remove_owned_drive = sub {
2255         my ($ds, $drive) = @_;
2256         return if drive_is_cdrom($drive, 1);
2257
2258         my $volid = $drive->{file};
2259         return if !$volid || $volid =~ m|^/|;
2260         return if $volids->{$volid};
2261
2262         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2263         return if !$path || !$owner || ($owner != $vmid);
2264
2265         $volids->{$volid} = 1;
2266         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2267         warn "Could not remove disk '$volid', check manually: $@" if $@;
2268     };
2269
2270     # only remove disks owned by this VM (referenced in the config)
2271     my $include_opts = {
2272         include_unused => 1,
2273         extra_keys => ['vmstate'],
2274     };
2275     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2276
2277     for my $snap (values %{$conf->{snapshots}}) {
2278         next if !defined($snap->{vmstate});
2279         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2280         next if !defined($drive);
2281         $remove_owned_drive->('vmstate', $drive);
2282     }
2283
2284     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2285
2286     if ($purge_unreferenced) { # also remove unreferenced disk
2287         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2288         PVE::Storage::foreach_volid($vmdisks, sub {
2289             my ($volid, $sid, $volname, $d) = @_;
2290             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2291             warn $@ if $@;
2292         });
2293     }
2294
2295     eval { delete_ifaces_ipams_ips($conf, $vmid)};
2296     warn $@ if $@;
2297
2298     if (defined $replacement_conf) {
2299         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2300     } else {
2301         PVE::QemuConfig->destroy_config($vmid);
2302     }
2303 }
2304
2305 sub parse_vm_config {
2306     my ($filename, $raw, $strict) = @_;
2307
2308     return if !defined($raw);
2309
2310     my $res = {
2311         digest => Digest::SHA::sha1_hex($raw),
2312         snapshots => {},
2313         pending => {},
2314         cloudinit => {},
2315     };
2316
2317     my $handle_error = sub {
2318         my ($msg) = @_;
2319
2320         if ($strict) {
2321             die $msg;
2322         } else {
2323             warn $msg;
2324         }
2325     };
2326
2327     $filename =~ m|/qemu-server/(\d+)\.conf$|
2328         || die "got strange filename '$filename'";
2329
2330     my $vmid = $1;
2331
2332     my $conf = $res;
2333     my $descr;
2334     my $finish_description = sub {
2335         if (defined($descr)) {
2336             $descr =~ s/\s+$//;
2337             $conf->{description} = $descr;
2338         }
2339         $descr = undef;
2340     };
2341     my $section = '';
2342
2343     my @lines = split(/\n/, $raw);
2344     foreach my $line (@lines) {
2345         next if $line =~ m/^\s*$/;
2346
2347         if ($line =~ m/^\[PENDING\]\s*$/i) {
2348             $section = 'pending';
2349             $finish_description->();
2350             $conf = $res->{$section} = {};
2351             next;
2352         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2353             $section = 'cloudinit';
2354             $finish_description->();
2355             $conf = $res->{$section} = {};
2356             next;
2357
2358         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2359             $section = $1;
2360             $finish_description->();
2361             $conf = $res->{snapshots}->{$section} = {};
2362             next;
2363         }
2364
2365         if ($line =~ m/^\#(.*)$/) {
2366             $descr = '' if !defined($descr);
2367             $descr .= PVE::Tools::decode_text($1) . "\n";
2368             next;
2369         }
2370
2371         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2372             $descr = '' if !defined($descr);
2373             $descr .= PVE::Tools::decode_text($2);
2374         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2375             $conf->{snapstate} = $1;
2376         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2377             my $key = $1;
2378             my $value = $2;
2379             $conf->{$key} = $value;
2380         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2381             my $value = $1;
2382             if ($section eq 'pending') {
2383                 $conf->{delete} = $value; # we parse this later
2384             } else {
2385                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2386             }
2387         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2388             my $key = $1;
2389             my $value = $2;
2390             if ($section eq 'cloudinit') {
2391                 # ignore validation only used for informative purpose
2392                 $conf->{$key} = $value;
2393                 next;
2394             }
2395             eval { $value = check_type($key, $value); };
2396             if ($@) {
2397                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2398             } else {
2399                 $key = 'ide2' if $key eq 'cdrom';
2400                 my $fmt = $confdesc->{$key}->{format};
2401                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2402                     my $v = parse_drive($key, $value);
2403                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2404                         $v->{file} = $volid;
2405                         $value = print_drive($v);
2406                     } else {
2407                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2408                         next;
2409                     }
2410                 }
2411
2412                 $conf->{$key} = $value;
2413             }
2414         } else {
2415             $handle_error->("vm $vmid - unable to parse config: $line\n");
2416         }
2417     }
2418
2419     $finish_description->();
2420     delete $res->{snapstate}; # just to be sure
2421
2422     return $res;
2423 }
2424
2425 sub write_vm_config {
2426     my ($filename, $conf) = @_;
2427
2428     delete $conf->{snapstate}; # just to be sure
2429
2430     if ($conf->{cdrom}) {
2431         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2432         $conf->{ide2} = $conf->{cdrom};
2433         delete $conf->{cdrom};
2434     }
2435
2436     # we do not use 'smp' any longer
2437     if ($conf->{sockets}) {
2438         delete $conf->{smp};
2439     } elsif ($conf->{smp}) {
2440         $conf->{sockets} = $conf->{smp};
2441         delete $conf->{cores};
2442         delete $conf->{smp};
2443     }
2444
2445     my $used_volids = {};
2446
2447     my $cleanup_config = sub {
2448         my ($cref, $pending, $snapname) = @_;
2449
2450         foreach my $key (keys %$cref) {
2451             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2452                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2453             my $value = $cref->{$key};
2454             if ($key eq 'delete') {
2455                 die "propertry 'delete' is only allowed in [PENDING]\n"
2456                     if !$pending;
2457                 # fixme: check syntax?
2458                 next;
2459             }
2460             eval { $value = check_type($key, $value); };
2461             die "unable to parse value of '$key' - $@" if $@;
2462
2463             $cref->{$key} = $value;
2464
2465             if (!$snapname && is_valid_drivename($key)) {
2466                 my $drive = parse_drive($key, $value);
2467                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2468             }
2469         }
2470     };
2471
2472     &$cleanup_config($conf);
2473
2474     &$cleanup_config($conf->{pending}, 1);
2475
2476     foreach my $snapname (keys %{$conf->{snapshots}}) {
2477         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2478         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2479     }
2480
2481     # remove 'unusedX' settings if we re-add a volume
2482     foreach my $key (keys %$conf) {
2483         my $value = $conf->{$key};
2484         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2485             delete $conf->{$key};
2486         }
2487     }
2488
2489     my $generate_raw_config = sub {
2490         my ($conf, $pending) = @_;
2491
2492         my $raw = '';
2493
2494         # add description as comment to top of file
2495         if (defined(my $descr = $conf->{description})) {
2496             if ($descr) {
2497                 foreach my $cl (split(/\n/, $descr)) {
2498                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2499                 }
2500             } else {
2501                 $raw .= "#\n" if $pending;
2502             }
2503         }
2504
2505         foreach my $key (sort keys %$conf) {
2506             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2507             $raw .= "$key: $conf->{$key}\n";
2508         }
2509         return $raw;
2510     };
2511
2512     my $raw = &$generate_raw_config($conf);
2513
2514     if (scalar(keys %{$conf->{pending}})){
2515         $raw .= "\n[PENDING]\n";
2516         $raw .= &$generate_raw_config($conf->{pending}, 1);
2517     }
2518
2519     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2520         $raw .= "\n[special:cloudinit]\n";
2521         $raw .= &$generate_raw_config($conf->{cloudinit});
2522     }
2523
2524     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2525         $raw .= "\n[$snapname]\n";
2526         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2527     }
2528
2529     return $raw;
2530 }
2531
2532 sub load_defaults {
2533
2534     my $res = {};
2535
2536     # we use static defaults from our JSON schema configuration
2537     foreach my $key (keys %$confdesc) {
2538         if (defined(my $default = $confdesc->{$key}->{default})) {
2539             $res->{$key} = $default;
2540         }
2541     }
2542
2543     return $res;
2544 }
2545
2546 sub config_list {
2547     my $vmlist = PVE::Cluster::get_vmlist();
2548     my $res = {};
2549     return $res if !$vmlist || !$vmlist->{ids};
2550     my $ids = $vmlist->{ids};
2551     my $nodename = nodename();
2552
2553     foreach my $vmid (keys %$ids) {
2554         my $d = $ids->{$vmid};
2555         next if !$d->{node} || $d->{node} ne $nodename;
2556         next if !$d->{type} || $d->{type} ne 'qemu';
2557         $res->{$vmid}->{exists} = 1;
2558     }
2559     return $res;
2560 }
2561
2562 # test if VM uses local resources (to prevent migration)
2563 sub check_local_resources {
2564     my ($conf, $noerr) = @_;
2565
2566     my @loc_res = ();
2567     my $mapped_res = [];
2568
2569     my $nodelist = PVE::Cluster::get_nodelist();
2570     my $pci_map = PVE::Mapping::PCI::config();
2571     my $usb_map = PVE::Mapping::USB::config();
2572
2573     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2574
2575     my $add_missing_mapping = sub {
2576         my ($type, $key, $id) = @_;
2577         for my $node (@$nodelist) {
2578             my $entry;
2579             if ($type eq 'pci') {
2580                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2581             } elsif ($type eq 'usb') {
2582                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2583             }
2584             if (!scalar($entry->@*)) {
2585                 push @{$missing_mappings_by_node->{$node}}, $key;
2586             }
2587         }
2588     };
2589
2590     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2591     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2592
2593     push @loc_res, "ivshmem" if $conf->{ivshmem};
2594
2595     foreach my $k (keys %$conf) {
2596         if ($k =~ m/^usb/) {
2597             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2598             next if $entry->{host} =~ m/^spice$/i;
2599             if ($entry->{mapping}) {
2600                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2601                 push @$mapped_res, $k;
2602             }
2603         }
2604         if ($k =~ m/^hostpci/) {
2605             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2606             if ($entry->{mapping}) {
2607                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2608                 push @$mapped_res, $k;
2609             }
2610         }
2611         # sockets are safe: they will recreated be on the target side post-migrate
2612         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2613         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2614     }
2615
2616     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2617
2618     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2619 }
2620
2621 # check if used storages are available on all nodes (use by migrate)
2622 sub check_storage_availability {
2623     my ($storecfg, $conf, $node) = @_;
2624
2625     PVE::QemuConfig->foreach_volume($conf, sub {
2626         my ($ds, $drive) = @_;
2627
2628         my $volid = $drive->{file};
2629         return if !$volid;
2630
2631         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2632         return if !$sid;
2633
2634         # check if storage is available on both nodes
2635         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2636         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2637
2638         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2639
2640         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2641             if !$scfg->{content}->{$vtype};
2642    });
2643 }
2644
2645 # list nodes where all VM images are available (used by has_feature API)
2646 sub shared_nodes {
2647     my ($conf, $storecfg) = @_;
2648
2649     my $nodelist = PVE::Cluster::get_nodelist();
2650     my $nodehash = { map { $_ => 1 } @$nodelist };
2651     my $nodename = nodename();
2652
2653     PVE::QemuConfig->foreach_volume($conf, sub {
2654         my ($ds, $drive) = @_;
2655
2656         my $volid = $drive->{file};
2657         return if !$volid;
2658
2659         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2660         if ($storeid) {
2661             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2662             if ($scfg->{disable}) {
2663                 $nodehash = {};
2664             } elsif (my $avail = $scfg->{nodes}) {
2665                 foreach my $node (keys %$nodehash) {
2666                     delete $nodehash->{$node} if !$avail->{$node};
2667                 }
2668             } elsif (!$scfg->{shared}) {
2669                 foreach my $node (keys %$nodehash) {
2670                     delete $nodehash->{$node} if $node ne $nodename
2671                 }
2672             }
2673         }
2674     });
2675
2676     return $nodehash
2677 }
2678
2679 sub check_local_storage_availability {
2680     my ($conf, $storecfg) = @_;
2681
2682     my $nodelist = PVE::Cluster::get_nodelist();
2683     my $nodehash = { map { $_ => {} } @$nodelist };
2684
2685     PVE::QemuConfig->foreach_volume($conf, sub {
2686         my ($ds, $drive) = @_;
2687
2688         my $volid = $drive->{file};
2689         return if !$volid;
2690
2691         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2692         if ($storeid) {
2693             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2694
2695             if ($scfg->{disable}) {
2696                 foreach my $node (keys %$nodehash) {
2697                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2698                 }
2699             } elsif (my $avail = $scfg->{nodes}) {
2700                 foreach my $node (keys %$nodehash) {
2701                     if (!$avail->{$node}) {
2702                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2703                     }
2704                 }
2705             }
2706         }
2707     });
2708
2709     foreach my $node (values %$nodehash) {
2710         if (my $unavail = $node->{unavailable_storages}) {
2711             $node->{unavailable_storages} = [ sort keys %$unavail ];
2712         }
2713     }
2714
2715     return $nodehash
2716 }
2717
2718 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2719 sub check_running {
2720     my ($vmid, $nocheck, $node) = @_;
2721
2722     # $nocheck is set when called during a migration, in which case the config
2723     # file might still or already reside on the *other* node
2724     # - because rename has already happened, and current node is source
2725     # - because rename hasn't happened yet, and current node is target
2726     # - because rename has happened, current node is target, but hasn't yet
2727     # processed it yet
2728     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2729     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2730 }
2731
2732 sub vzlist {
2733
2734     my $vzlist = config_list();
2735
2736     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2737
2738     while (defined(my $de = $fd->read)) {
2739         next if $de !~ m/^(\d+)\.pid$/;
2740         my $vmid = $1;
2741         next if !defined($vzlist->{$vmid});
2742         if (my $pid = check_running($vmid)) {
2743             $vzlist->{$vmid}->{pid} = $pid;
2744         }
2745     }
2746
2747     return $vzlist;
2748 }
2749
2750 our $vmstatus_return_properties = {
2751     vmid => get_standard_option('pve-vmid'),
2752     status => {
2753         description => "QEMU process status.",
2754         type => 'string',
2755         enum => ['stopped', 'running'],
2756     },
2757     maxmem => {
2758         description => "Maximum memory in bytes.",
2759         type => 'integer',
2760         optional => 1,
2761         renderer => 'bytes',
2762     },
2763     maxdisk => {
2764         description => "Root disk size in bytes.",
2765         type => 'integer',
2766         optional => 1,
2767         renderer => 'bytes',
2768     },
2769     name => {
2770         description => "VM name.",
2771         type => 'string',
2772         optional => 1,
2773     },
2774     qmpstatus => {
2775         description => "VM run state from the 'query-status' QMP monitor command.",
2776         type => 'string',
2777         optional => 1,
2778     },
2779     pid => {
2780         description => "PID of running qemu process.",
2781         type => 'integer',
2782         optional => 1,
2783     },
2784     uptime => {
2785         description => "Uptime.",
2786         type => 'integer',
2787         optional => 1,
2788         renderer => 'duration',
2789     },
2790     cpus => {
2791         description => "Maximum usable CPUs.",
2792         type => 'number',
2793         optional => 1,
2794     },
2795     lock => {
2796         description => "The current config lock, if any.",
2797         type => 'string',
2798         optional => 1,
2799     },
2800     tags => {
2801         description  => "The current configured tags, if any",
2802         type => 'string',
2803         optional => 1,
2804     },
2805     'running-machine' => {
2806         description => "The currently running machine type (if running).",
2807         type => 'string',
2808         optional => 1,
2809     },
2810     'running-qemu' => {
2811         description => "The currently running QEMU version (if running).",
2812         type => 'string',
2813         optional => 1,
2814     },
2815 };
2816
2817 my $last_proc_pid_stat;
2818
2819 # get VM status information
2820 # This must be fast and should not block ($full == false)
2821 # We only query KVM using QMP if $full == true (this can be slow)
2822 sub vmstatus {
2823     my ($opt_vmid, $full) = @_;
2824
2825     my $res = {};
2826
2827     my $storecfg = PVE::Storage::config();
2828
2829     my $list = vzlist();
2830     my $defaults = load_defaults();
2831
2832     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2833
2834     my $cpucount = $cpuinfo->{cpus} || 1;
2835
2836     foreach my $vmid (keys %$list) {
2837         next if $opt_vmid && ($vmid ne $opt_vmid);
2838
2839         my $conf = PVE::QemuConfig->load_config($vmid);
2840
2841         my $d = { vmid => int($vmid) };
2842         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2843
2844         # fixme: better status?
2845         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2846
2847         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2848         if (defined($size)) {
2849             $d->{disk} = 0; # no info available
2850             $d->{maxdisk} = $size;
2851         } else {
2852             $d->{disk} = 0;
2853             $d->{maxdisk} = 0;
2854         }
2855
2856         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2857             * ($conf->{cores} || $defaults->{cores});
2858         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2859         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2860
2861         $d->{name} = $conf->{name} || "VM $vmid";
2862         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2863
2864         if ($conf->{balloon}) {
2865             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2866             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2867                 : $defaults->{shares};
2868         }
2869
2870         $d->{uptime} = 0;
2871         $d->{cpu} = 0;
2872         $d->{mem} = 0;
2873
2874         $d->{netout} = 0;
2875         $d->{netin} = 0;
2876
2877         $d->{diskread} = 0;
2878         $d->{diskwrite} = 0;
2879
2880         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2881
2882         $d->{serial} = 1 if conf_has_serial($conf);
2883         $d->{lock} = $conf->{lock} if $conf->{lock};
2884         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2885
2886         $res->{$vmid} = $d;
2887     }
2888
2889     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2890     foreach my $dev (keys %$netdev) {
2891         next if $dev !~ m/^tap([1-9]\d*)i/;
2892         my $vmid = $1;
2893         my $d = $res->{$vmid};
2894         next if !$d;
2895
2896         $d->{netout} += $netdev->{$dev}->{receive};
2897         $d->{netin} += $netdev->{$dev}->{transmit};
2898
2899         if ($full) {
2900             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2901             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2902         }
2903
2904     }
2905
2906     my $ctime = gettimeofday;
2907
2908     foreach my $vmid (keys %$list) {
2909
2910         my $d = $res->{$vmid};
2911         my $pid = $d->{pid};
2912         next if !$pid;
2913
2914         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2915         next if !$pstat; # not running
2916
2917         my $used = $pstat->{utime} + $pstat->{stime};
2918
2919         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2920
2921         if ($pstat->{vsize}) {
2922             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2923         }
2924
2925         my $old = $last_proc_pid_stat->{$pid};
2926         if (!$old) {
2927             $last_proc_pid_stat->{$pid} = {
2928                 time => $ctime,
2929                 used => $used,
2930                 cpu => 0,
2931             };
2932             next;
2933         }
2934
2935         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2936
2937         if ($dtime > 1000) {
2938             my $dutime = $used -  $old->{used};
2939
2940             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2941             $last_proc_pid_stat->{$pid} = {
2942                 time => $ctime,
2943                 used => $used,
2944                 cpu => $d->{cpu},
2945             };
2946         } else {
2947             $d->{cpu} = $old->{cpu};
2948         }
2949     }
2950
2951     return $res if !$full;
2952
2953     my $qmpclient = PVE::QMPClient->new();
2954
2955     my $ballooncb = sub {
2956         my ($vmid, $resp) = @_;
2957
2958         my $info = $resp->{'return'};
2959         return if !$info->{max_mem};
2960
2961         my $d = $res->{$vmid};
2962
2963         # use memory assigned to VM
2964         $d->{maxmem} = $info->{max_mem};
2965         $d->{balloon} = $info->{actual};
2966
2967         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2968             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2969             $d->{freemem} = $info->{free_mem};
2970         }
2971
2972         $d->{ballooninfo} = $info;
2973     };
2974
2975     my $blockstatscb = sub {
2976         my ($vmid, $resp) = @_;
2977         my $data = $resp->{'return'} || [];
2978         my $totalrdbytes = 0;
2979         my $totalwrbytes = 0;
2980
2981         for my $blockstat (@$data) {
2982             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2983             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2984
2985             $blockstat->{device} =~ s/drive-//;
2986             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2987         }
2988         $res->{$vmid}->{diskread} = $totalrdbytes;
2989         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2990     };
2991
2992     my $machinecb = sub {
2993         my ($vmid, $resp) = @_;
2994         my $data = $resp->{'return'} || [];
2995
2996         $res->{$vmid}->{'running-machine'} =
2997             PVE::QemuServer::Machine::current_from_query_machines($data);
2998     };
2999
3000     my $versioncb = sub {
3001         my ($vmid, $resp) = @_;
3002         my $data = $resp->{'return'} // {};
3003         my $version = 'unknown';
3004
3005         if (my $v = $data->{qemu}) {
3006             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3007         }
3008
3009         $res->{$vmid}->{'running-qemu'} = $version;
3010     };
3011
3012     my $statuscb = sub {
3013         my ($vmid, $resp) = @_;
3014
3015         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3016         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3017         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3018         # this fails if ballon driver is not loaded, so this must be
3019         # the last commnand (following command are aborted if this fails).
3020         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3021
3022         my $status = 'unknown';
3023         if (!defined($status = $resp->{'return'}->{status})) {
3024             warn "unable to get VM status\n";
3025             return;
3026         }
3027
3028         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3029     };
3030
3031     foreach my $vmid (keys %$list) {
3032         next if $opt_vmid && ($vmid ne $opt_vmid);
3033         next if !$res->{$vmid}->{pid}; # not running
3034         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3035     }
3036
3037     $qmpclient->queue_execute(undef, 2);
3038
3039     foreach my $vmid (keys %$list) {
3040         next if $opt_vmid && ($vmid ne $opt_vmid);
3041         next if !$res->{$vmid}->{pid}; #not running
3042
3043         # we can't use the $qmpclient since it might have already aborted on
3044         # 'query-balloon', but this might also fail for older versions...
3045         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3046         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3047     }
3048
3049     foreach my $vmid (keys %$list) {
3050         next if $opt_vmid && ($vmid ne $opt_vmid);
3051         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3052     }
3053
3054     return $res;
3055 }
3056
3057 sub conf_has_serial {
3058     my ($conf) = @_;
3059
3060     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3061         if ($conf->{"serial$i"}) {
3062             return 1;
3063         }
3064     }
3065
3066     return 0;
3067 }
3068
3069 sub conf_has_audio {
3070     my ($conf, $id) = @_;
3071
3072     $id //= 0;
3073     my $audio = $conf->{"audio$id"};
3074     return if !defined($audio);
3075
3076     my $audioproperties = parse_property_string($audio_fmt, $audio);
3077     my $audiodriver = $audioproperties->{driver} // 'spice';
3078
3079     return {
3080         dev => $audioproperties->{device},
3081         dev_id => "audiodev$id",
3082         backend => $audiodriver,
3083         backend_id => "$audiodriver-backend${id}",
3084     };
3085 }
3086
3087 sub audio_devs {
3088     my ($audio, $audiopciaddr, $machine_version) = @_;
3089
3090     my $devs = [];
3091
3092     my $id = $audio->{dev_id};
3093     my $audiodev = "";
3094     if (min_version($machine_version, 4, 2)) {
3095         $audiodev = ",audiodev=$audio->{backend_id}";
3096     }
3097
3098     if ($audio->{dev} eq 'AC97') {
3099         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3100     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3101         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3102         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3103         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3104     } else {
3105         die "unkown audio device '$audio->{dev}', implement me!";
3106     }
3107
3108     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3109
3110     return $devs;
3111 }
3112
3113 sub get_tpm_paths {
3114     my ($vmid) = @_;
3115     return {
3116         socket => "/var/run/qemu-server/$vmid.swtpm",
3117         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3118     };
3119 }
3120
3121 sub add_tpm_device {
3122     my ($vmid, $devices, $conf) = @_;
3123
3124     return if !$conf->{tpmstate0};
3125
3126     my $paths = get_tpm_paths($vmid);
3127
3128     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3129     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3130     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3131 }
3132
3133 sub start_swtpm {
3134     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3135
3136     return if !$tpmdrive;
3137
3138     my $state;
3139     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3140     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3141     if ($storeid) {
3142         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3143     } else {
3144         $state = $tpm->{file};
3145     }
3146
3147     my $paths = get_tpm_paths($vmid);
3148
3149     # during migration, we will get state from remote
3150     #
3151     if (!$migration) {
3152         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3153         my $setup_cmd = [
3154             "swtpm_setup",
3155             "--tpmstate",
3156             "file://$state",
3157             "--createek",
3158             "--create-ek-cert",
3159             "--create-platform-cert",
3160             "--lock-nvram",
3161             "--config",
3162             "/etc/swtpm_setup.conf", # do not use XDG configs
3163             "--runas",
3164             "0", # force creation as root, error if not possible
3165             "--not-overwrite", # ignore existing state, do not modify
3166         ];
3167
3168         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3169         # TPM 2.0 supports ECC crypto, use if possible
3170         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3171
3172         run_command($setup_cmd, outfunc => sub {
3173             print "swtpm_setup: $1\n";
3174         });
3175     }
3176
3177     # Used to distinguish different invocations in the log.
3178     my $log_prefix = "[id=" . int(time()) . "] ";
3179
3180     my $emulator_cmd = [
3181         "swtpm",
3182         "socket",
3183         "--tpmstate",
3184         "backend-uri=file://$state,mode=0600",
3185         "--ctrl",
3186         "type=unixio,path=$paths->{socket},mode=0600",
3187         "--pid",
3188         "file=$paths->{pid}",
3189         "--terminate", # terminate on QEMU disconnect
3190         "--daemon",
3191         "--log",
3192         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3193     ];
3194     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3195     run_command($emulator_cmd, outfunc => sub { print $1; });
3196
3197     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3198     while (! -e $paths->{pid}) {
3199         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3200         usleep(50_000);
3201     }
3202
3203     # return untainted PID of swtpm daemon so it can be killed on error
3204     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3205     return $1;
3206 }
3207
3208 sub vga_conf_has_spice {
3209     my ($vga) = @_;
3210
3211     my $vgaconf = parse_vga($vga);
3212     my $vgatype = $vgaconf->{type};
3213     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3214
3215     return $1 || 1;
3216 }
3217
3218 sub is_native($) {
3219     my ($arch) = @_;
3220     return get_host_arch() eq $arch;
3221 }
3222
3223 sub get_vm_arch {
3224     my ($conf) = @_;
3225     return $conf->{arch} // get_host_arch();
3226 }
3227
3228 my $default_machines = {
3229     x86_64 => 'pc',
3230     aarch64 => 'virt',
3231 };
3232
3233 sub get_installed_machine_version {
3234     my ($kvmversion) = @_;
3235     $kvmversion = kvm_user_version() if !defined($kvmversion);
3236     $kvmversion =~ m/^(\d+\.\d+)/;
3237     return $1;
3238 }
3239
3240 sub windows_get_pinned_machine_version {
3241     my ($machine, $base_version, $kvmversion) = @_;
3242
3243     my $pin_version = $base_version;
3244     if (!defined($base_version) ||
3245         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3246     ) {
3247         $pin_version = get_installed_machine_version($kvmversion);
3248     }
3249     if (!$machine || $machine eq 'pc') {
3250         $machine = "pc-i440fx-$pin_version";
3251     } elsif ($machine eq 'q35') {
3252         $machine = "pc-q35-$pin_version";
3253     } elsif ($machine eq 'virt') {
3254         $machine = "virt-$pin_version";
3255     } else {
3256         warn "unknown machine type '$machine', not touching that!\n";
3257     }
3258
3259     return $machine;
3260 }
3261
3262 sub get_vm_machine {
3263     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3264
3265     my $machine = $forcemachine || $conf->{machine};
3266
3267     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3268         $kvmversion //= kvm_user_version();
3269         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3270         # layout which confuses windows quite a bit and may result in various regressions..
3271         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3272         if (windows_version($conf->{ostype})) {
3273             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3274         }
3275         $arch //= 'x86_64';
3276         $machine ||= $default_machines->{$arch};
3277         if ($add_pve_version) {
3278             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3279             $machine .= "+pve$pvever";
3280         }
3281     }
3282
3283     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3284         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3285         $machine = $1 if $is_pxe;
3286
3287         # for version-pinned machines that do not include a pve-version (e.g.
3288         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3289         $machine .= '+pve0';
3290
3291         $machine .= '.pxe' if $is_pxe;
3292     }
3293
3294     return $machine;
3295 }
3296
3297 sub get_ovmf_files($$$) {
3298     my ($arch, $efidisk, $smm) = @_;
3299
3300     my $types = $OVMF->{$arch}
3301         or die "no OVMF images known for architecture '$arch'\n";
3302
3303     my $type = 'default';
3304     if ($arch eq 'x86_64') {
3305         if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3306             $type = $smm ? "4m" : "4m-no-smm";
3307             $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3308         } else {
3309             # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3310         }
3311     }
3312
3313     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3314     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3315     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3316
3317     return ($ovmf_code, $ovmf_vars);
3318 }
3319
3320 my $Arch2Qemu = {
3321     aarch64 => '/usr/bin/qemu-system-aarch64',
3322     x86_64 => '/usr/bin/qemu-system-x86_64',
3323 };
3324 sub get_command_for_arch($) {
3325     my ($arch) = @_;
3326     return '/usr/bin/kvm' if is_native($arch);
3327
3328     my $cmd = $Arch2Qemu->{$arch}
3329         or die "don't know how to emulate architecture '$arch'\n";
3330     return $cmd;
3331 }
3332
3333 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3334 # to use in a QEMU command line (-cpu element), first array_intersect the result
3335 # of query_supported_ with query_understood_. This is necessary because:
3336 #
3337 # a) query_understood_ returns flags the host cannot use and
3338 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3339 #    flags, but CPU settings - with most of them being flags. Those settings
3340 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3341 #
3342 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3343 # expensive. If you need the value returned from this, you can get it much
3344 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3345 # $accel being 'kvm' or 'tcg'.
3346 #
3347 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3348 # changes, automatically populating pmxcfs.
3349 #
3350 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3351 # since kvm and tcg machines support different flags
3352 #
3353 sub query_supported_cpu_flags {
3354     my ($arch) = @_;
3355
3356     $arch //= get_host_arch();
3357     my $default_machine = $default_machines->{$arch};
3358
3359     my $flags = {};
3360
3361     # FIXME: Once this is merged, the code below should work for ARM as well:
3362     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3363     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3364         $arch eq "aarch64";
3365
3366     my $kvm_supported = defined(kvm_version());
3367     my $qemu_cmd = get_command_for_arch($arch);
3368     my $fakevmid = -1;
3369     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3370
3371     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3372     my $query_supported_run_qemu = sub {
3373         my ($kvm) = @_;
3374
3375         my $flags = {};
3376         my $cmd = [
3377             $qemu_cmd,
3378             '-machine', $default_machine,
3379             '-display', 'none',
3380             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3381             '-mon', 'chardev=qmp,mode=control',
3382             '-pidfile', $pidfile,
3383             '-S', '-daemonize'
3384         ];
3385
3386         if (!$kvm) {
3387             push @$cmd, '-accel', 'tcg';
3388         }
3389
3390         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3391         die "QEMU flag querying VM exited with code " . $rc if $rc;
3392
3393         eval {
3394             my $cmd_result = mon_cmd(
3395                 $fakevmid,
3396                 'query-cpu-model-expansion',
3397                 type => 'full',
3398                 model => { name => 'host' }
3399             );
3400
3401             my $props = $cmd_result->{model}->{props};
3402             foreach my $prop (keys %$props) {
3403                 next if $props->{$prop} ne '1';
3404                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3405                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3406                 # We only keep those with underscores, to match /proc/cpuinfo
3407                 $prop =~ s/\.|-/_/g;
3408                 $flags->{$prop} = 1;
3409             }
3410         };
3411         my $err = $@;
3412
3413         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3414         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3415
3416         die $err if $err;
3417
3418         return [ sort keys %$flags ];
3419     };
3420
3421     # We need to query QEMU twice, since KVM and TCG have different supported flags
3422     PVE::QemuConfig->lock_config($fakevmid, sub {
3423         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3424         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3425
3426         if ($kvm_supported) {
3427             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3428             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3429         }
3430     });
3431
3432     return $flags;
3433 }
3434
3435 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3436 my $understood_cpu_flag_dir = "/usr/share/kvm";
3437 sub query_understood_cpu_flags {
3438     my $arch = get_host_arch();
3439     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3440
3441     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3442         if ! -e $filepath;
3443
3444     my $raw = file_get_contents($filepath);
3445     $raw =~ s/^\s+|\s+$//g;
3446     my @flags = split(/\s+/, $raw);
3447
3448     return \@flags;
3449 }
3450
3451 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3452 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3453 my sub should_disable_smm {
3454     my ($conf, $vga, $machine) = @_;
3455
3456     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3457
3458     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3459         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3460 }
3461
3462 my sub print_ovmf_drive_commandlines {
3463     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3464
3465     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3466
3467     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3468
3469     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3470     if ($d) {
3471         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3472         my ($path, $format) = $d->@{'file', 'format'};
3473         if ($storeid) {
3474             $path = PVE::Storage::path($storecfg, $d->{file});
3475             if (!defined($format)) {
3476                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3477                 $format = qemu_img_format($scfg, $volname);
3478             }
3479         } elsif (!defined($format)) {
3480             die "efidisk format must be specified\n";
3481         }
3482         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3483         if ($path =~ m/^rbd:/) {
3484             $var_drive_str .= ',cache=writeback';
3485             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3486         }
3487         $var_drive_str .= ",format=$format,file=$path";
3488
3489         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3490         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3491     } else {
3492         log_warn("no efidisk configured! Using temporary efivars disk.");
3493         my $path = "/tmp/$vmid-ovmf.fd";
3494         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3495         $var_drive_str .= ",format=raw,file=$path";
3496         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3497     }
3498
3499     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3500 }
3501
3502 sub config_to_command {
3503     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3504         $pbs_backing) = @_;
3505
3506     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3507     my $devices = [];
3508     my $bridges = {};
3509     my $ostype = $conf->{ostype};
3510     my $winversion = windows_version($ostype);
3511     my $kvm = $conf->{kvm};
3512     my $nodename = nodename();
3513
3514     my $arch = get_vm_arch($conf);
3515     my $kvm_binary = get_command_for_arch($arch);
3516     my $kvmver = kvm_user_version($kvm_binary);
3517
3518     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3519         $kvmver //= "undefined";
3520         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3521     }
3522
3523     my $add_pve_version = min_version($kvmver, 4, 1);
3524
3525     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3526     my $machine_version = extract_version($machine_type, $kvmver);
3527     $kvm //= 1 if is_native($arch);
3528
3529     $machine_version =~ m/(\d+)\.(\d+)/;
3530     my ($machine_major, $machine_minor) = ($1, $2);
3531
3532     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3533         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3534     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3535         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3536             ." please upgrade node '$nodename'\n"
3537     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3538         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3539         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3540             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3541             ." node '$nodename'\n";
3542     }
3543
3544     # if a specific +pve version is required for a feature, use $version_guard
3545     # instead of min_version to allow machines to be run with the minimum
3546     # required version
3547     my $required_pve_version = 0;
3548     my $version_guard = sub {
3549         my ($major, $minor, $pve) = @_;
3550         return 0 if !min_version($machine_version, $major, $minor, $pve);
3551         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3552         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3553         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3554         return 1;
3555     };
3556
3557     if ($kvm && !defined kvm_version()) {
3558         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3559             ." or enable in BIOS.\n";
3560     }
3561
3562     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3563     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3564     my $use_old_bios_files = undef;
3565     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3566
3567     my $cmd = [];
3568     if ($conf->{affinity}) {
3569         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3570     }
3571
3572     push @$cmd, $kvm_binary;
3573
3574     push @$cmd, '-id', $vmid;
3575
3576     my $vmname = $conf->{name} || "vm$vmid";
3577
3578     push @$cmd, '-name', "$vmname,debug-threads=on";
3579
3580     push @$cmd, '-no-shutdown';
3581
3582     my $use_virtio = 0;
3583
3584     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3585     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3586     push @$cmd, '-mon', "chardev=qmp,mode=control";
3587
3588     if (min_version($machine_version, 2, 12)) {
3589         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3590         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3591     }
3592
3593     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3594
3595     push @$cmd, '-daemonize';
3596
3597     if ($conf->{smbios1}) {
3598         my $smbios_conf = parse_smbios1($conf->{smbios1});
3599         if ($smbios_conf->{base64}) {
3600             # Do not pass base64 flag to qemu
3601             delete $smbios_conf->{base64};
3602             my $smbios_string = "";
3603             foreach my $key (keys %$smbios_conf) {
3604                 my $value;
3605                 if ($key eq "uuid") {
3606                     $value = $smbios_conf->{uuid}
3607                 } else {
3608                     $value = decode_base64($smbios_conf->{$key});
3609                 }
3610                 # qemu accepts any binary data, only commas need escaping by double comma
3611                 $value =~ s/,/,,/g;
3612                 $smbios_string .= "," . $key . "=" . $value if $value;
3613             }
3614             push @$cmd, '-smbios', "type=1" . $smbios_string;
3615         } else {
3616             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3617         }
3618     }
3619
3620     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3621         die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n"
3622             if !$forcecpu && get_cpu_bitness($conf->{cpu}, $arch) == 32;
3623
3624         my ($code_drive_str, $var_drive_str) =
3625             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3626         push $cmd->@*, '-drive', $code_drive_str;
3627         push $cmd->@*, '-drive', $var_drive_str;
3628     }
3629
3630     if ($q35) { # tell QEMU to load q35 config early
3631         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3632         if (min_version($machine_version, 4, 0)) {
3633             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3634         } else {
3635             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3636         }
3637     }
3638
3639     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3640         push @$cmd, $fixups->@*;
3641     }
3642
3643     if ($conf->{vmgenid}) {
3644         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3645     }
3646
3647     # add usb controllers
3648     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3649         $conf, $bridges, $arch, $machine_type, $machine_version);
3650     push @$devices, @usbcontrollers if @usbcontrollers;
3651     my $vga = parse_vga($conf->{vga});
3652
3653     my $qxlnum = vga_conf_has_spice($conf->{vga});
3654     $vga->{type} = 'qxl' if $qxlnum;
3655
3656     if (!$vga->{type}) {
3657         if ($arch eq 'aarch64') {
3658             $vga->{type} = 'virtio';
3659         } elsif (min_version($machine_version, 2, 9)) {
3660             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3661         } else {
3662             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3663         }
3664     }
3665
3666     # enable absolute mouse coordinates (needed by vnc)
3667     my $tablet = $conf->{tablet};
3668     if (!defined($tablet)) {
3669         $tablet = $defaults->{tablet};
3670         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3671         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3672     }
3673
3674     if ($tablet) {
3675         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3676         my $kbd = print_keyboarddevice_full($conf, $arch);
3677         push @$devices, '-device', $kbd if defined($kbd);
3678     }
3679
3680     my $bootorder = device_bootorder($conf);
3681
3682     # host pci device passthrough
3683     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3684         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3685
3686     # usb devices
3687     my $usb_dev_features = {};
3688     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3689
3690     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3691         $conf, $usb_dev_features, $bootorder, $machine_version);
3692     push @$devices, @usbdevices if @usbdevices;
3693
3694     # serial devices
3695     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3696         my $path = $conf->{"serial$i"} or next;
3697         if ($path eq 'socket') {
3698             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3699             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3700             # On aarch64, serial0 is the UART device. QEMU only allows
3701             # connecting UART devices via the '-serial' command line, as
3702             # the device has a fixed slot on the hardware...
3703             if ($arch eq 'aarch64' && $i == 0) {
3704                 push @$devices, '-serial', "chardev:serial$i";
3705             } else {
3706                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3707             }
3708         } else {
3709             die "no such serial device\n" if ! -c $path;
3710             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3711             push @$devices, '-device', "isa-serial,chardev=serial$i";
3712         }
3713     }
3714
3715     # parallel devices
3716     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3717         if (my $path = $conf->{"parallel$i"}) {
3718             die "no such parallel device\n" if ! -c $path;
3719             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3720             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3721             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3722         }
3723     }
3724
3725     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3726         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3727         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3728         push @$devices, @$audio_devs;
3729     }
3730
3731     # Add a TPM only if the VM is not a template,
3732     # to support backing up template VMs even if the TPM disk is write-protected.
3733     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3734
3735     my $sockets = 1;
3736     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3737     $sockets = $conf->{sockets} if  $conf->{sockets};
3738
3739     my $cores = $conf->{cores} || 1;
3740
3741     my $maxcpus = $sockets * $cores;
3742
3743     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3744
3745     my $allowed_vcpus = $cpuinfo->{cpus};
3746
3747     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3748
3749     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3750         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3751         for (my $i = 2; $i <= $vcpus; $i++)  {
3752             my $cpustr = print_cpu_device($conf,$i);
3753             push @$cmd, '-device', $cpustr;
3754         }
3755
3756     } else {
3757
3758         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3759     }
3760     push @$cmd, '-nodefaults';
3761
3762     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3763
3764     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3765
3766     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3767
3768     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3769         push @$devices, '-device', print_vga_device(
3770             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3771
3772         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3773
3774         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3775         push @$cmd,  '-vnc', "unix:$socket,password=on";
3776     } else {
3777         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3778         push @$cmd, '-nographic';
3779     }
3780
3781     # time drift fix
3782     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3783     my $useLocaltime = $conf->{localtime};
3784
3785     if ($winversion >= 5) { # windows
3786         $useLocaltime = 1 if !defined($conf->{localtime});
3787
3788         # use time drift fix when acpi is enabled
3789         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3790             $tdf = 1 if !defined($conf->{tdf});
3791         }
3792     }
3793
3794     if ($winversion >= 6) {
3795         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3796         push @$machineFlags, 'hpet=off';
3797     }
3798
3799     push @$rtcFlags, 'driftfix=slew' if $tdf;
3800
3801     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3802         push @$rtcFlags, "base=$conf->{startdate}";
3803     } elsif ($useLocaltime) {
3804         push @$rtcFlags, 'base=localtime';
3805     }
3806
3807     if ($forcecpu) {
3808         push @$cmd, '-cpu', $forcecpu;
3809     } else {
3810         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3811     }
3812
3813     PVE::QemuServer::Memory::config(
3814         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3815
3816     push @$cmd, '-S' if $conf->{freeze};
3817
3818     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3819
3820     my $guest_agent = parse_guest_agent($conf);
3821
3822     if ($guest_agent->{enabled}) {
3823         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3824         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3825
3826         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3827             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3828             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3829             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3830         } elsif ($guest_agent->{type} eq 'isa') {
3831             push @$devices, '-device', "isa-serial,chardev=qga0";
3832         }
3833     }
3834
3835     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3836     if ($rng && $version_guard->(4, 1, 2)) {
3837         check_rng_source($rng->{source});
3838
3839         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3840         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3841         my $limiter_str = "";
3842         if ($max_bytes) {
3843             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3844         }
3845
3846         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3847         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3848         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3849     }
3850
3851     my $spice_port;
3852
3853     assert_clipboard_config($vga);
3854     my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3855
3856     if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3857         if ($qxlnum > 1) {
3858             if ($winversion){
3859                 for (my $i = 1; $i < $qxlnum; $i++){
3860                     push @$devices, '-device', print_vga_device(
3861                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3862                 }
3863             } else {
3864                 # assume other OS works like Linux
3865                 my ($ram, $vram) = ("134217728", "67108864");
3866                 if ($vga->{memory}) {
3867                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3868                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3869                 }
3870                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3871                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3872             }
3873         }
3874
3875         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3876
3877         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3878         if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3879             push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3880         } else {
3881             push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3882         }
3883         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3884
3885         if ($is_spice) {
3886             my $pfamily = PVE::Tools::get_host_address_family($nodename);
3887             my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3888             die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3889
3890             my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3891             $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3892
3893             my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3894             my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3895             if ($spice_enhancement->{foldersharing}) {
3896                 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3897                 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3898             }
3899
3900             my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3901             $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3902                 if $spice_enhancement->{videostreaming};
3903             push @$devices, '-spice', "$spice_opts";
3904         }
3905     }
3906
3907     # enable balloon by default, unless explicitly disabled
3908     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3909         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3910         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3911         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3912         push @$devices, '-device', $ballooncmd;
3913     }
3914
3915     if ($conf->{watchdog}) {
3916         my $wdopts = parse_watchdog($conf->{watchdog});
3917         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3918         my $watchdog = $wdopts->{model} || 'i6300esb';
3919         push @$devices, '-device', "$watchdog$pciaddr";
3920         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3921     }
3922
3923     my $vollist = [];
3924     my $scsicontroller = {};
3925     my $ahcicontroller = {};
3926     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3927
3928     # Add iscsi initiator name if available
3929     if (my $initiator = get_initiator_name()) {
3930         push @$devices, '-iscsi', "initiator-name=$initiator";
3931     }
3932
3933     PVE::QemuConfig->foreach_volume($conf, sub {
3934         my ($ds, $drive) = @_;
3935
3936         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3937             check_volume_storage_type($storecfg, $drive->{file});
3938             push @$vollist, $drive->{file};
3939         }
3940
3941         # ignore efidisk here, already added in bios/fw handling code above
3942         return if $drive->{interface} eq 'efidisk';
3943         # similar for TPM
3944         return if $drive->{interface} eq 'tpmstate';
3945
3946         $use_virtio = 1 if $ds =~ m/^virtio/;
3947
3948         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3949
3950         if ($drive->{interface} eq 'virtio'){
3951            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3952         }
3953
3954         if ($drive->{interface} eq 'scsi') {
3955
3956             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3957
3958             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3959                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3960
3961             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3962             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3963
3964             my $iothread = '';
3965             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3966                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3967                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3968             } elsif ($drive->{iothread}) {
3969                 log_warn(
3970                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3971                 );
3972             }
3973
3974             my $queues = '';
3975             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3976                 $queues = ",num_queues=$drive->{queues}";
3977             }
3978
3979             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3980                 if !$scsicontroller->{$controller};
3981             $scsicontroller->{$controller}=1;
3982         }
3983
3984         if ($drive->{interface} eq 'sata') {
3985             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3986             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3987             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3988                 if !$ahcicontroller->{$controller};
3989             $ahcicontroller->{$controller}=1;
3990         }
3991
3992         my $pbs_conf = $pbs_backing->{$ds};
3993         my $pbs_name = undef;
3994         if ($pbs_conf) {
3995             $pbs_name = "drive-$ds-pbs";
3996             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3997         }
3998
3999         my $drive_cmd = print_drive_commandline_full(
4000             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4001
4002         # extra protection for templates, but SATA and IDE don't support it..
4003         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4004
4005         push @$devices, '-drive',$drive_cmd;
4006         push @$devices, '-device', print_drivedevice_full(
4007             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4008     });
4009
4010     for (my $i = 0; $i < $MAX_NETS; $i++) {
4011         my $netname = "net$i";
4012
4013         next if !$conf->{$netname};
4014         my $d = parse_net($conf->{$netname});
4015         next if !$d;
4016         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4017
4018         $use_virtio = 1 if $d->{model} eq 'virtio';
4019
4020         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4021
4022         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4023         push @$devices, '-netdev', $netdevfull;
4024
4025         my $netdevicefull = print_netdevice_full(
4026             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4027
4028         push @$devices, '-device', $netdevicefull;
4029     }
4030
4031     if ($conf->{ivshmem}) {
4032         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4033
4034         my $bus;
4035         if ($q35) {
4036             $bus = print_pcie_addr("ivshmem");
4037         } else {
4038             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4039         }
4040
4041         my $ivshmem_name = $ivshmem->{name} // $vmid;
4042         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4043
4044         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4045         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4046             .",size=$ivshmem->{size}M";
4047     }
4048
4049     # pci.4 is nested in pci.1
4050     $bridges->{1} = 1 if $bridges->{4};
4051
4052     if (!$q35) { # add pci bridges
4053         if (min_version($machine_version, 2, 3)) {
4054            $bridges->{1} = 1;
4055            $bridges->{2} = 1;
4056         }
4057         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4058     }
4059
4060     for my $k (sort {$b cmp $a} keys %$bridges) {
4061         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4062
4063         my $k_name = $k;
4064         if ($k == 2 && $legacy_igd) {
4065             $k_name = "$k-igd";
4066         }
4067         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4068         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4069
4070         if ($q35) { # add after -readconfig pve-q35.cfg
4071             splice @$devices, 2, 0, '-device', $devstr;
4072         } else {
4073             unshift @$devices, '-device', $devstr if $k > 0;
4074         }
4075     }
4076
4077     if (!$kvm) {
4078         push @$machineFlags, 'accel=tcg';
4079     }
4080
4081     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4082
4083     my $machine_type_min = $machine_type;
4084     if ($add_pve_version) {
4085         $machine_type_min =~ s/\+pve\d+$//;
4086         $machine_type_min .= "+pve$required_pve_version";
4087     }
4088     push @$machineFlags, "type=${machine_type_min}";
4089
4090     push @$cmd, @$devices;
4091     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4092     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4093     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4094
4095     if (my $vmstate = $conf->{vmstate}) {
4096         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4097         push @$vollist, $vmstate;
4098         push @$cmd, '-loadstate', $statepath;
4099         print "activating and using '$vmstate' as vmstate\n";
4100     }
4101
4102     if (PVE::QemuConfig->is_template($conf)) {
4103         # needed to workaround base volumes being read-only
4104         push @$cmd, '-snapshot';
4105     }
4106
4107     # add custom args
4108     if ($conf->{args}) {
4109         my $aa = PVE::Tools::split_args($conf->{args});
4110         push @$cmd, @$aa;
4111     }
4112
4113     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4114 }
4115
4116 sub check_rng_source {
4117     my ($source) = @_;
4118
4119     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4120     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4121         if ! -e $source;
4122
4123     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4124     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4125         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4126         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4127         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4128             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4129             ." to the host.\n";
4130     }
4131 }
4132
4133 sub spice_port {
4134     my ($vmid) = @_;
4135
4136     my $res = mon_cmd($vmid, 'query-spice');
4137
4138     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4139 }
4140
4141 sub vm_devices_list {
4142     my ($vmid) = @_;
4143
4144     my $res = mon_cmd($vmid, 'query-pci');
4145     my $devices_to_check = [];
4146     my $devices = {};
4147     foreach my $pcibus (@$res) {
4148         push @$devices_to_check, @{$pcibus->{devices}},
4149     }
4150
4151     while (@$devices_to_check) {
4152         my $to_check = [];
4153         for my $d (@$devices_to_check) {
4154             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4155             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4156
4157             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4158             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4159         }
4160         $devices_to_check = $to_check;
4161     }
4162
4163     my $resblock = mon_cmd($vmid, 'query-block');
4164     foreach my $block (@$resblock) {
4165         if($block->{device} =~ m/^drive-(\S+)/){
4166                 $devices->{$1} = 1;
4167         }
4168     }
4169
4170     my $resmice = mon_cmd($vmid, 'query-mice');
4171     foreach my $mice (@$resmice) {
4172         if ($mice->{name} eq 'QEMU HID Tablet') {
4173             $devices->{tablet} = 1;
4174             last;
4175         }
4176     }
4177
4178     # for usb devices there is no query-usb
4179     # but we can iterate over the entries in
4180     # qom-list path=/machine/peripheral
4181     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4182     foreach my $per (@$resperipheral) {
4183         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4184             $devices->{$per->{name}} = 1;
4185         }
4186     }
4187
4188     return $devices;
4189 }
4190
4191 sub vm_deviceplug {
4192     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4193
4194     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4195
4196     my $devices_list = vm_devices_list($vmid);
4197     return 1 if defined($devices_list->{$deviceid});
4198
4199     # add PCI bridge if we need it for the device
4200     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4201
4202     if ($deviceid eq 'tablet') {
4203         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4204     } elsif ($deviceid eq 'keyboard') {
4205         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4206     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4207         my $id = $1;
4208         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4209         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4210     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4211         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4212     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4213         qemu_iothread_add($vmid, $deviceid, $device);
4214
4215         qemu_driveadd($storecfg, $vmid, $device);
4216         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4217
4218         qemu_deviceadd($vmid, $devicefull);
4219         eval { qemu_deviceaddverify($vmid, $deviceid); };
4220         if (my $err = $@) {
4221             eval { qemu_drivedel($vmid, $deviceid); };
4222             warn $@ if $@;
4223             die $err;
4224         }
4225     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4226         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4227         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4228         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4229
4230         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4231
4232         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4233             qemu_iothread_add($vmid, $deviceid, $device);
4234             $devicefull .= ",iothread=iothread-$deviceid";
4235         }
4236
4237         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4238             $devicefull .= ",num_queues=$device->{queues}";
4239         }
4240
4241         qemu_deviceadd($vmid, $devicefull);
4242         qemu_deviceaddverify($vmid, $deviceid);
4243     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4244         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4245         qemu_driveadd($storecfg, $vmid, $device);
4246
4247         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4248         eval { qemu_deviceadd($vmid, $devicefull); };
4249         if (my $err = $@) {
4250             eval { qemu_drivedel($vmid, $deviceid); };
4251             warn $@ if $@;
4252             die $err;
4253         }
4254     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4255         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4256
4257         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4258         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4259         my $use_old_bios_files = undef;
4260         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4261
4262         my $netdevicefull = print_netdevice_full(
4263             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4264         qemu_deviceadd($vmid, $netdevicefull);
4265         eval {
4266             qemu_deviceaddverify($vmid, $deviceid);
4267             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4268         };
4269         if (my $err = $@) {
4270             eval { qemu_netdevdel($vmid, $deviceid); };
4271             warn $@ if $@;
4272             die $err;
4273         }
4274     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4275         my $bridgeid = $2;
4276         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4277         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4278
4279         qemu_deviceadd($vmid, $devicefull);
4280         qemu_deviceaddverify($vmid, $deviceid);
4281     } else {
4282         die "can't hotplug device '$deviceid'\n";
4283     }
4284
4285     return 1;
4286 }
4287
4288 # fixme: this should raise exceptions on error!
4289 sub vm_deviceunplug {
4290     my ($vmid, $conf, $deviceid) = @_;
4291
4292     my $devices_list = vm_devices_list($vmid);
4293     return 1 if !defined($devices_list->{$deviceid});
4294
4295     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4296     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4297
4298     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4299         qemu_devicedel($vmid, $deviceid);
4300     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4301         qemu_devicedel($vmid, $deviceid);
4302         qemu_devicedelverify($vmid, $deviceid);
4303     } elsif ($deviceid =~ m/^usb\d+$/) {
4304         qemu_devicedel($vmid, $deviceid);
4305         qemu_devicedelverify($vmid, $deviceid);
4306     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4307         my $device = parse_drive($deviceid, $conf->{$deviceid});
4308
4309         qemu_devicedel($vmid, $deviceid);
4310         qemu_devicedelverify($vmid, $deviceid);
4311         qemu_drivedel($vmid, $deviceid);
4312         qemu_iothread_del($vmid, $deviceid, $device);
4313     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4314         qemu_devicedel($vmid, $deviceid);
4315         qemu_devicedelverify($vmid, $deviceid);
4316     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4317         my $device = parse_drive($deviceid, $conf->{$deviceid});
4318
4319         qemu_devicedel($vmid, $deviceid);
4320         qemu_devicedelverify($vmid, $deviceid);
4321         qemu_drivedel($vmid, $deviceid);
4322         qemu_deletescsihw($conf, $vmid, $deviceid);
4323
4324         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4325             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4326     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4327         qemu_devicedel($vmid, $deviceid);
4328         qemu_devicedelverify($vmid, $deviceid);
4329         qemu_netdevdel($vmid, $deviceid);
4330     } else {
4331         die "can't unplug device '$deviceid'\n";
4332     }
4333
4334     return 1;
4335 }
4336
4337 sub qemu_spice_usbredir_chardev_add {
4338     my ($vmid, $id) = @_;
4339
4340     mon_cmd($vmid, "chardev-add" , (
4341         id => $id,
4342         backend => {
4343             type => 'spicevmc',
4344             data => {
4345                 type => "usbredir",
4346             },
4347         },
4348     ));
4349 }
4350
4351 sub qemu_iothread_add {
4352     my ($vmid, $deviceid, $device) = @_;
4353
4354     if ($device->{iothread}) {
4355         my $iothreads = vm_iothreads_list($vmid);
4356         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4357     }
4358 }
4359
4360 sub qemu_iothread_del {
4361     my ($vmid, $deviceid, $device) = @_;
4362
4363     if ($device->{iothread}) {
4364         my $iothreads = vm_iothreads_list($vmid);
4365         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4366     }
4367 }
4368
4369 sub qemu_driveadd {
4370     my ($storecfg, $vmid, $device) = @_;
4371
4372     my $kvmver = get_running_qemu_version($vmid);
4373     my $io_uring = min_version($kvmver, 6, 0);
4374     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4375     $drive =~ s/\\/\\\\/g;
4376     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4377
4378     # If the command succeeds qemu prints: "OK"
4379     return 1 if $ret =~ m/OK/s;
4380
4381     die "adding drive failed: $ret\n";
4382 }
4383
4384 sub qemu_drivedel {
4385     my ($vmid, $deviceid) = @_;
4386
4387     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4388     $ret =~ s/^\s+//;
4389
4390     return 1 if $ret eq "";
4391
4392     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4393     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4394
4395     die "deleting drive $deviceid failed : $ret\n";
4396 }
4397
4398 sub qemu_deviceaddverify {
4399     my ($vmid, $deviceid) = @_;
4400
4401     for (my $i = 0; $i <= 5; $i++) {
4402          my $devices_list = vm_devices_list($vmid);
4403          return 1 if defined($devices_list->{$deviceid});
4404          sleep 1;
4405     }
4406
4407     die "error on hotplug device '$deviceid'\n";
4408 }
4409
4410
4411 sub qemu_devicedelverify {
4412     my ($vmid, $deviceid) = @_;
4413
4414     # need to verify that the device is correctly removed as device_del
4415     # is async and empty return is not reliable
4416
4417     for (my $i = 0; $i <= 5; $i++) {
4418          my $devices_list = vm_devices_list($vmid);
4419          return 1 if !defined($devices_list->{$deviceid});
4420          sleep 1;
4421     }
4422
4423     die "error on hot-unplugging device '$deviceid'\n";
4424 }
4425
4426 sub qemu_findorcreatescsihw {
4427     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4428
4429     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4430
4431     my $scsihwid="$controller_prefix$controller";
4432     my $devices_list = vm_devices_list($vmid);
4433
4434     if (!defined($devices_list->{$scsihwid})) {
4435         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4436     }
4437
4438     return 1;
4439 }
4440
4441 sub qemu_deletescsihw {
4442     my ($conf, $vmid, $opt) = @_;
4443
4444     my $device = parse_drive($opt, $conf->{$opt});
4445
4446     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4447         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4448         return 1;
4449     }
4450
4451     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4452
4453     my $devices_list = vm_devices_list($vmid);
4454     foreach my $opt (keys %{$devices_list}) {
4455         if (is_valid_drivename($opt)) {
4456             my $drive = parse_drive($opt, $conf->{$opt});
4457             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4458                 return 1;
4459             }
4460         }
4461     }
4462
4463     my $scsihwid="scsihw$controller";
4464
4465     vm_deviceunplug($vmid, $conf, $scsihwid);
4466
4467     return 1;
4468 }
4469
4470 sub qemu_add_pci_bridge {
4471     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4472
4473     my $bridges = {};
4474
4475     my $bridgeid;
4476
4477     print_pci_addr($device, $bridges, $arch, $machine_type);
4478
4479     while (my ($k, $v) = each %$bridges) {
4480         $bridgeid = $k;
4481     }
4482     return 1 if !defined($bridgeid) || $bridgeid < 1;
4483
4484     my $bridge = "pci.$bridgeid";
4485     my $devices_list = vm_devices_list($vmid);
4486
4487     if (!defined($devices_list->{$bridge})) {
4488         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4489     }
4490
4491     return 1;
4492 }
4493
4494 sub qemu_set_link_status {
4495     my ($vmid, $device, $up) = @_;
4496
4497     mon_cmd($vmid, "set_link", name => $device,
4498                up => $up ? JSON::true : JSON::false);
4499 }
4500
4501 sub qemu_netdevadd {
4502     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4503
4504     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4505     my %options =  split(/[=,]/, $netdev);
4506
4507     if (defined(my $vhost = $options{vhost})) {
4508         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4509     }
4510
4511     if (defined(my $queues = $options{queues})) {
4512         $options{queues} = $queues + 0;
4513     }
4514
4515     mon_cmd($vmid, "netdev_add",  %options);
4516     return 1;
4517 }
4518
4519 sub qemu_netdevdel {
4520     my ($vmid, $deviceid) = @_;
4521
4522     mon_cmd($vmid, "netdev_del", id => $deviceid);
4523 }
4524
4525 sub qemu_usb_hotplug {
4526     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4527
4528     return if !$device;
4529
4530     # remove the old one first
4531     vm_deviceunplug($vmid, $conf, $deviceid);
4532
4533     # check if xhci controller is necessary and available
4534     my $devicelist = vm_devices_list($vmid);
4535
4536     if (!$devicelist->{xhci}) {
4537         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4538         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4539     }
4540
4541     # add the new one
4542     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4543 }
4544
4545 sub qemu_cpu_hotplug {
4546     my ($vmid, $conf, $vcpus) = @_;
4547
4548     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4549
4550     my $sockets = 1;
4551     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4552     $sockets = $conf->{sockets} if  $conf->{sockets};
4553     my $cores = $conf->{cores} || 1;
4554     my $maxcpus = $sockets * $cores;
4555
4556     $vcpus = $maxcpus if !$vcpus;
4557
4558     die "you can't add more vcpus than maxcpus\n"
4559         if $vcpus > $maxcpus;
4560
4561     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4562
4563     if ($vcpus < $currentvcpus) {
4564
4565         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4566
4567             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4568                 qemu_devicedel($vmid, "cpu$i");
4569                 my $retry = 0;
4570                 my $currentrunningvcpus = undef;
4571                 while (1) {
4572                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4573                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4574                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4575                     $retry++;
4576                     sleep 1;
4577                 }
4578                 #update conf after each succesfull cpu unplug
4579                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4580                 PVE::QemuConfig->write_config($vmid, $conf);
4581             }
4582         } else {
4583             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4584         }
4585
4586         return;
4587     }
4588
4589     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4590     die "vcpus in running vm does not match its configuration\n"
4591         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4592
4593     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4594
4595         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4596             my $cpustr = print_cpu_device($conf, $i);
4597             qemu_deviceadd($vmid, $cpustr);
4598
4599             my $retry = 0;
4600             my $currentrunningvcpus = undef;
4601             while (1) {
4602                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4603                 last if scalar(@{$currentrunningvcpus}) == $i;
4604                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4605                 sleep 1;
4606                 $retry++;
4607             }
4608             #update conf after each succesfull cpu hotplug
4609             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4610             PVE::QemuConfig->write_config($vmid, $conf);
4611         }
4612     } else {
4613
4614         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4615             mon_cmd($vmid, "cpu-add", id => int($i));
4616         }
4617     }
4618 }
4619
4620 sub qemu_block_set_io_throttle {
4621     my ($vmid, $deviceid,
4622         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4623         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4624         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4625         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4626
4627     return if !check_running($vmid) ;
4628
4629     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4630         bps => int($bps),
4631         bps_rd => int($bps_rd),
4632         bps_wr => int($bps_wr),
4633         iops => int($iops),
4634         iops_rd => int($iops_rd),
4635         iops_wr => int($iops_wr),
4636         bps_max => int($bps_max),
4637         bps_rd_max => int($bps_rd_max),
4638         bps_wr_max => int($bps_wr_max),
4639         iops_max => int($iops_max),
4640         iops_rd_max => int($iops_rd_max),
4641         iops_wr_max => int($iops_wr_max),
4642         bps_max_length => int($bps_max_length),
4643         bps_rd_max_length => int($bps_rd_max_length),
4644         bps_wr_max_length => int($bps_wr_max_length),
4645         iops_max_length => int($iops_max_length),
4646         iops_rd_max_length => int($iops_rd_max_length),
4647         iops_wr_max_length => int($iops_wr_max_length),
4648     );
4649
4650 }
4651
4652 sub qemu_block_resize {
4653     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4654
4655     my $running = check_running($vmid);
4656
4657     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4658
4659     return if !$running;
4660
4661     my $padding = (1024 - $size % 1024) % 1024;
4662     $size = $size + $padding;
4663
4664     mon_cmd(
4665         $vmid,
4666         "block_resize",
4667         device => $deviceid,
4668         size => int($size),
4669         timeout => 60,
4670     );
4671 }
4672
4673 sub qemu_volume_snapshot {
4674     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4675
4676     my $running = check_running($vmid);
4677
4678     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4679         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4680     } else {
4681         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4682     }
4683 }
4684
4685 sub qemu_volume_snapshot_delete {
4686     my ($vmid, $storecfg, $volid, $snap) = @_;
4687
4688     my $running = check_running($vmid);
4689     my $attached_deviceid;
4690
4691     if ($running) {
4692         my $conf = PVE::QemuConfig->load_config($vmid);
4693         PVE::QemuConfig->foreach_volume($conf, sub {
4694             my ($ds, $drive) = @_;
4695             $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4696         });
4697     }
4698
4699     if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4700         mon_cmd(
4701             $vmid,
4702             'blockdev-snapshot-delete-internal-sync',
4703             device => $attached_deviceid,
4704             name => $snap,
4705         );
4706     } else {
4707         PVE::Storage::volume_snapshot_delete(
4708             $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4709     }
4710 }
4711
4712 sub set_migration_caps {
4713     my ($vmid, $savevm) = @_;
4714
4715     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4716
4717     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4718     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4719
4720     my $cap_ref = [];
4721
4722     my $enabled_cap = {
4723         "auto-converge" => 1,
4724         "xbzrle" => 1,
4725         "x-rdma-pin-all" => 0,
4726         "zero-blocks" => 0,
4727         "compress" => 0,
4728         "dirty-bitmaps" => $dirty_bitmaps,
4729     };
4730
4731     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4732
4733     for my $supported_capability (@$supported_capabilities) {
4734         push @$cap_ref, {
4735             capability => $supported_capability->{capability},
4736             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4737         };
4738     }
4739
4740     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4741 }
4742
4743 sub foreach_volid {
4744     my ($conf, $func, @param) = @_;
4745
4746     my $volhash = {};
4747
4748     my $test_volid = sub {
4749         my ($key, $drive, $snapname, $pending) = @_;
4750
4751         my $volid = $drive->{file};
4752         return if !$volid;
4753
4754         $volhash->{$volid}->{cdrom} //= 1;
4755         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4756
4757         my $replicate = $drive->{replicate} // 1;
4758         $volhash->{$volid}->{replicate} //= 0;
4759         $volhash->{$volid}->{replicate} = 1 if $replicate;
4760
4761         $volhash->{$volid}->{shared} //= 0;
4762         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4763
4764         $volhash->{$volid}->{is_unused} //= 0;
4765         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4766
4767         $volhash->{$volid}->{is_attached} //= 0;
4768         $volhash->{$volid}->{is_attached} = 1
4769             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4770
4771         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4772             if defined($snapname);
4773
4774         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4775
4776         my $size = $drive->{size};
4777         $volhash->{$volid}->{size} //= $size if $size;
4778
4779         $volhash->{$volid}->{is_vmstate} //= 0;
4780         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4781
4782         $volhash->{$volid}->{is_tpmstate} //= 0;
4783         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4784
4785         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4786     };
4787
4788     my $include_opts = {
4789         extra_keys => ['vmstate'],
4790         include_unused => 1,
4791     };
4792
4793     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4794
4795     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4796         if defined($conf->{pending}) && $conf->{pending}->%*;
4797
4798     foreach my $snapname (keys %{$conf->{snapshots}}) {
4799         my $snap = $conf->{snapshots}->{$snapname};
4800         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4801     }
4802
4803     foreach my $volid (keys %$volhash) {
4804         &$func($volid, $volhash->{$volid}, @param);
4805     }
4806 }
4807
4808 my $fast_plug_option = {
4809     'description' => 1,
4810     'hookscript' => 1,
4811     'lock' => 1,
4812     'migrate_downtime' => 1,
4813     'migrate_speed' => 1,
4814     'name' => 1,
4815     'onboot' => 1,
4816     'protection' => 1,
4817     'shares' => 1,
4818     'startup' => 1,
4819     'tags' => 1,
4820     'vmstatestorage' => 1,
4821 };
4822
4823 for my $opt (keys %$confdesc_cloudinit) {
4824     $fast_plug_option->{$opt} = 1;
4825 };
4826
4827 # hotplug changes in [PENDING]
4828 # $selection hash can be used to only apply specified options, for
4829 # example: { cores => 1 } (only apply changed 'cores')
4830 # $errors ref is used to return error messages
4831 sub vmconfig_hotplug_pending {
4832     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4833
4834     my $defaults = load_defaults();
4835     my $arch = get_vm_arch($conf);
4836     my $machine_type = get_vm_machine($conf, undef, $arch);
4837
4838     # commit values which do not have any impact on running VM first
4839     # Note: those option cannot raise errors, we we do not care about
4840     # $selection and always apply them.
4841
4842     my $add_error = sub {
4843         my ($opt, $msg) = @_;
4844         $errors->{$opt} = "hotplug problem - $msg";
4845     };
4846
4847     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4848
4849     my $cloudinit_record_changed = sub {
4850         my ($conf, $opt, $old, $new) = @_;
4851         return if !$cloudinit_pending_properties->{$opt};
4852
4853         my $ci = ($conf->{cloudinit} //= {});
4854
4855         my $recorded = $ci->{$opt};
4856         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4857
4858         if (defined($new)) {
4859             if (defined($old)) {
4860                 # an existing value is being modified
4861                 if (defined($recorded)) {
4862                     # the value was already not in sync
4863                     if ($new eq $recorded) {
4864                         # a value is being reverted to the cloud-init state:
4865                         delete $ci->{$opt};
4866                         delete $added{$opt};
4867                     } else {
4868                         # the value was changed multiple times, do nothing
4869                     }
4870                 } elsif ($added{$opt}) {
4871                     # the value had been marked as added and is being changed, do nothing
4872                 } else {
4873                     # the value is new, record it:
4874                     $ci->{$opt} = $old;
4875                 }
4876             } else {
4877                 # a new value is being added
4878                 if (defined($recorded)) {
4879                     # it was already not in sync
4880                     if ($new eq $recorded) {
4881                         # a value is being reverted to the cloud-init state:
4882                         delete $ci->{$opt};
4883                         delete $added{$opt};
4884                     } else {
4885                         # the value had temporarily been removed, do nothing
4886                     }
4887                 } elsif ($added{$opt}) {
4888                     # the value had been marked as added already, do nothing
4889                 } else {
4890                     # the value is new, add it
4891                     $added{$opt} = 1;
4892                 }
4893             }
4894         } elsif (!defined($old)) {
4895             # a non-existent value is being removed? ignore...
4896         } else {
4897             # a value is being deleted
4898             if (defined($recorded)) {
4899                 # a value was already recorded, just keep it
4900             } elsif ($added{$opt}) {
4901                 # the value was marked as added, remove it
4902                 delete $added{$opt};
4903             } else {
4904                 # a previously unrecorded value is being removed, record the old value:
4905                 $ci->{$opt} = $old;
4906             }
4907         }
4908
4909         my $added = join(',', sort keys %added);
4910         $ci->{added} = $added if length($added);
4911     };
4912
4913     my $changes = 0;
4914     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4915         if ($fast_plug_option->{$opt}) {
4916             my $new = delete $conf->{pending}->{$opt};
4917             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4918             $conf->{$opt} = $new;
4919             $changes = 1;
4920         }
4921     }
4922
4923     if ($changes) {
4924         PVE::QemuConfig->write_config($vmid, $conf);
4925     }
4926
4927     my $ostype = $conf->{ostype};
4928     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4929     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4930     my $usb_hotplug = $hotplug_features->{usb}
4931         && min_version($version, 7, 1)
4932         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4933
4934     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4935     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4936
4937     foreach my $opt (sort keys %$pending_delete_hash) {
4938         next if $selection && !$selection->{$opt};
4939         my $force = $pending_delete_hash->{$opt}->{force};
4940         eval {
4941             if ($opt eq 'hotplug') {
4942                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4943             } elsif ($opt eq 'tablet') {
4944                 die "skip\n" if !$hotplug_features->{usb};
4945                 if ($defaults->{tablet}) {
4946                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4947                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4948                         if $arch eq 'aarch64';
4949                 } else {
4950                     vm_deviceunplug($vmid, $conf, 'tablet');
4951                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4952                 }
4953             } elsif ($opt =~ m/^usb(\d+)$/) {
4954                 my $index = $1;
4955                 die "skip\n" if !$usb_hotplug;
4956                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4957                 vm_deviceunplug($vmid, $conf, $opt);
4958             } elsif ($opt eq 'vcpus') {
4959                 die "skip\n" if !$hotplug_features->{cpu};
4960                 qemu_cpu_hotplug($vmid, $conf, undef);
4961             } elsif ($opt eq 'balloon') {
4962                 # enable balloon device is not hotpluggable
4963                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4964                 # here we reset the ballooning value to memory
4965                 my $balloon = get_current_memory($conf->{memory});
4966                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4967             } elsif ($fast_plug_option->{$opt}) {
4968                 # do nothing
4969             } elsif ($opt =~ m/^net(\d+)$/) {
4970                 die "skip\n" if !$hotplug_features->{network};
4971                 vm_deviceunplug($vmid, $conf, $opt);
4972                 if($have_sdn) {
4973                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
4974                     PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
4975                 }
4976             } elsif (is_valid_drivename($opt)) {
4977                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4978                 vm_deviceunplug($vmid, $conf, $opt);
4979                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4980             } elsif ($opt =~ m/^memory$/) {
4981                 die "skip\n" if !$hotplug_features->{memory};
4982                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
4983             } elsif ($opt eq 'cpuunits') {
4984                 $cgroup->change_cpu_shares(undef);
4985             } elsif ($opt eq 'cpulimit') {
4986                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4987             } else {
4988                 die "skip\n";
4989             }
4990         };
4991         if (my $err = $@) {
4992             &$add_error($opt, $err) if $err ne "skip\n";
4993         } else {
4994             my $old = delete $conf->{$opt};
4995             $cloudinit_record_changed->($conf, $opt, $old, undef);
4996             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4997         }
4998     }
4999
5000     my $cloudinit_opt;
5001     foreach my $opt (keys %{$conf->{pending}}) {
5002         next if $selection && !$selection->{$opt};
5003         my $value = $conf->{pending}->{$opt};
5004         eval {
5005             if ($opt eq 'hotplug') {
5006                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5007                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
5008             } elsif ($opt eq 'tablet') {
5009                 die "skip\n" if !$hotplug_features->{usb};
5010                 if ($value == 1) {
5011                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5012                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5013                         if $arch eq 'aarch64';
5014                 } elsif ($value == 0) {
5015                     vm_deviceunplug($vmid, $conf, 'tablet');
5016                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5017                 }
5018             } elsif ($opt =~ m/^usb(\d+)$/) {
5019                 my $index = $1;
5020                 die "skip\n" if !$usb_hotplug;
5021                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5022                 my $id = $opt;
5023                 if ($d->{host} =~ m/^spice$/i)  {
5024                     $id = "usbredirdev$index";
5025                 }
5026                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5027             } elsif ($opt eq 'vcpus') {
5028                 die "skip\n" if !$hotplug_features->{cpu};
5029                 qemu_cpu_hotplug($vmid, $conf, $value);
5030             } elsif ($opt eq 'balloon') {
5031                 # enable/disable balloning device is not hotpluggable
5032                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5033                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5034                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5035
5036                 # allow manual ballooning if shares is set to zero
5037                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5038                     my $memory = get_current_memory($conf->{memory});
5039                     my $balloon = $conf->{pending}->{balloon} || $memory;
5040                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5041                 }
5042             } elsif ($opt =~ m/^net(\d+)$/) {
5043                 # some changes can be done without hotplug
5044                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5045                                     $vmid, $opt, $value, $arch, $machine_type);
5046             } elsif (is_valid_drivename($opt)) {
5047                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5048                 # some changes can be done without hotplug
5049                 my $drive = parse_drive($opt, $value);
5050                 if (drive_is_cloudinit($drive)) {
5051                     $cloudinit_opt = [$opt, $drive];
5052                     # apply all the other changes first, then generate the cloudinit disk
5053                     die "skip\n";
5054                 }
5055                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5056                                      $vmid, $opt, $value, $arch, $machine_type);
5057             } elsif ($opt =~ m/^memory$/) { #dimms
5058                 die "skip\n" if !$hotplug_features->{memory};
5059                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5060             } elsif ($opt eq 'cpuunits') {
5061                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5062                 $cgroup->change_cpu_shares($new_cpuunits);
5063             } elsif ($opt eq 'cpulimit') {
5064                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5065                 $cgroup->change_cpu_quota($cpulimit, 100000);
5066             } elsif ($opt eq 'agent') {
5067                 vmconfig_update_agent($conf, $opt, $value);
5068             } else {
5069                 die "skip\n";  # skip non-hot-pluggable options
5070             }
5071         };
5072         if (my $err = $@) {
5073             &$add_error($opt, $err) if $err ne "skip\n";
5074         } else {
5075             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5076             $conf->{$opt} = $value;
5077             delete $conf->{pending}->{$opt};
5078         }
5079     }
5080
5081     if (defined($cloudinit_opt)) {
5082         my ($opt, $drive) = @$cloudinit_opt;
5083         my $value = $conf->{pending}->{$opt};
5084         eval {
5085             my $temp = {%$conf, $opt => $value};
5086             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5087             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5088                                  $vmid, $opt, $value, $arch, $machine_type);
5089         };
5090         if (my $err = $@) {
5091             &$add_error($opt, $err) if $err ne "skip\n";
5092         } else {
5093             $conf->{$opt} = $value;
5094             delete $conf->{pending}->{$opt};
5095         }
5096     }
5097
5098     # unplug xhci controller if no usb device is left
5099     if ($usb_hotplug) {
5100         my $has_usb = 0;
5101         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5102             next if !defined($conf->{"usb$i"});
5103             $has_usb = 1;
5104             last;
5105         }
5106         if (!$has_usb) {
5107             vm_deviceunplug($vmid, $conf, 'xhci');
5108         }
5109     }
5110
5111     PVE::QemuConfig->write_config($vmid, $conf);
5112
5113     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5114         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5115     }
5116 }
5117
5118 sub try_deallocate_drive {
5119     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5120
5121     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5122         my $volid = $drive->{file};
5123         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5124             my $sid = PVE::Storage::parse_volume_id($volid);
5125             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5126
5127             # check if the disk is really unused
5128             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5129                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5130             PVE::Storage::vdisk_free($storecfg, $volid);
5131             return 1;
5132         } else {
5133             # If vm is not owner of this disk remove from config
5134             return 1;
5135         }
5136     }
5137
5138     return;
5139 }
5140
5141 sub vmconfig_delete_or_detach_drive {
5142     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5143
5144     my $drive = parse_drive($opt, $conf->{$opt});
5145
5146     my $rpcenv = PVE::RPCEnvironment::get();
5147     my $authuser = $rpcenv->get_user();
5148
5149     if ($force) {
5150         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5151         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5152     } else {
5153         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5154     }
5155 }
5156
5157
5158
5159 sub vmconfig_apply_pending {
5160     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5161
5162     return if !scalar(keys %{$conf->{pending}});
5163
5164     my $add_apply_error = sub {
5165         my ($opt, $msg) = @_;
5166         my $err_msg = "unable to apply pending change $opt : $msg";
5167         $errors->{$opt} = $err_msg;
5168         warn $err_msg;
5169     };
5170
5171     # cold plug
5172
5173     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5174     foreach my $opt (sort keys %$pending_delete_hash) {
5175         my $force = $pending_delete_hash->{$opt}->{force};
5176         eval {
5177             if ($opt =~ m/^unused/) {
5178                 die "internal error";
5179             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5180                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5181             } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5182                 if($have_sdn) {
5183                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5184                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5185                     warn if $@;
5186                 }
5187             }
5188         };
5189         if (my $err = $@) {
5190             $add_apply_error->($opt, $err);
5191         } else {
5192             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5193             delete $conf->{$opt};
5194         }
5195     }
5196
5197     PVE::QemuConfig->cleanup_pending($conf);
5198
5199     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5200
5201     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5202         next if $opt eq 'delete'; # just to be sure
5203         eval {
5204             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5205                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5206             } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5207                 if($have_sdn) {
5208                     my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5209                     if ($conf->{$opt}){
5210                         my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5211
5212                         if ($old_net->{bridge} ne $new_net->{bridge} ||
5213                             $old_net->{macaddr} ne $new_net->{macaddr}) {
5214                             PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5215                         }
5216                    }
5217                    #fixme: reuse ip if mac change && same bridge
5218                    PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5219                 }
5220             }
5221         };
5222         if (my $err = $@) {
5223             $add_apply_error->($opt, $err);
5224         } else {
5225
5226             if (is_valid_drivename($opt)) {
5227                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5228                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5229             }
5230
5231             $conf->{$opt} = delete $conf->{pending}->{$opt};
5232         }
5233     }
5234
5235     # write all changes at once to avoid unnecessary i/o
5236     PVE::QemuConfig->write_config($vmid, $conf);
5237     if ($generate_cloudinit) {
5238         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5239             # After successful generation and if there were changes to be applied, update the
5240             # config to drop the {cloudinit} entry.
5241             PVE::QemuConfig->write_config($vmid, $conf);
5242         }
5243     }
5244 }
5245
5246 sub vmconfig_update_net {
5247     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5248
5249     my $newnet = parse_net($value);
5250
5251     if ($conf->{$opt}) {
5252         my $oldnet = parse_net($conf->{$opt});
5253
5254         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5255             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5256             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5257             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5258             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5259
5260             # for non online change, we try to hot-unplug
5261             die "skip\n" if !$hotplug;
5262             vm_deviceunplug($vmid, $conf, $opt);
5263
5264             if($have_sdn) {
5265                 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5266             }
5267
5268         } else {
5269
5270             die "internal error" if $opt !~ m/net(\d+)/;
5271             my $iface = "tap${vmid}i$1";
5272
5273             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5274                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5275                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5276                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5277                 PVE::Network::tap_unplug($iface);
5278
5279                 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5280                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5281                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5282                     qemu_set_link_status($vmid, $opt, 0);
5283                 }
5284
5285                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5286                     if ($have_sdn) {
5287                         PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5288                         PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5289                     }
5290                 }
5291
5292                 if ($have_sdn) {
5293                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5294                 } else {
5295                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5296                 }
5297
5298                 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5299                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5300                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5301                     qemu_set_link_status($vmid, $opt, 1);
5302                 }
5303
5304             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5305                 # Rate can be applied on its own but any change above needs to
5306                 # include the rate in tap_plug since OVS resets everything.
5307                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5308             }
5309
5310             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5311                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5312             }
5313
5314             return 1;
5315         }
5316     }
5317
5318     if ($hotplug) {
5319         if ($have_sdn) {
5320             PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5321             PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5322         }
5323         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5324     } else {
5325         die "skip\n";
5326     }
5327 }
5328
5329 sub vmconfig_update_agent {
5330     my ($conf, $opt, $value) = @_;
5331
5332     die "skip\n" if !$conf->{$opt};
5333
5334     my $hotplug_options = { fstrim_cloned_disks => 1 };
5335
5336     my $old_agent = parse_guest_agent($conf);
5337     my $agent = parse_guest_agent({$opt => $value});
5338
5339     for my $option (keys %$agent) { # added/changed options
5340         next if defined($hotplug_options->{$option});
5341         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5342     }
5343
5344     for my $option (keys %$old_agent) { # removed options
5345         next if defined($hotplug_options->{$option});
5346         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5347     }
5348
5349     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5350 }
5351
5352 sub vmconfig_update_disk {
5353     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5354
5355     my $drive = parse_drive($opt, $value);
5356
5357     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5358         my $media = $drive->{media} || 'disk';
5359         my $oldmedia = $old_drive->{media} || 'disk';
5360         die "unable to change media type\n" if $media ne $oldmedia;
5361
5362         if (!drive_is_cdrom($old_drive)) {
5363
5364             if ($drive->{file} ne $old_drive->{file}) {
5365
5366                 die "skip\n" if !$hotplug;
5367
5368                 # unplug and register as unused
5369                 vm_deviceunplug($vmid, $conf, $opt);
5370                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5371
5372             } else {
5373                 # update existing disk
5374
5375                 # skip non hotpluggable value
5376                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5377                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5378                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5379                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5380                     safe_string_ne($drive->{product}, $old_drive->{product}) ||
5381                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5382                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5383                     safe_string_ne($drive->{vendor}, $old_drive->{vendor}) ||
5384                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5385                     die "skip\n";
5386                 }
5387
5388                 # apply throttle
5389                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5390                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5391                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5392                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5393                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5394                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5395                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5396                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5397                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5398                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5399                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5400                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5401                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5402                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5403                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5404                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5405                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5406                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5407
5408                     qemu_block_set_io_throttle(
5409                         $vmid,"drive-$opt",
5410                         ($drive->{mbps} || 0)*1024*1024,
5411                         ($drive->{mbps_rd} || 0)*1024*1024,
5412                         ($drive->{mbps_wr} || 0)*1024*1024,
5413                          $drive->{iops} || 0,
5414                          $drive->{iops_rd} || 0,
5415                          $drive->{iops_wr} || 0,
5416                         ($drive->{mbps_max} || 0)*1024*1024,
5417                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5418                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5419                          $drive->{iops_max} || 0,
5420                          $drive->{iops_rd_max} || 0,
5421                          $drive->{iops_wr_max} || 0,
5422                          $drive->{bps_max_length} || 1,
5423                          $drive->{bps_rd_max_length} || 1,
5424                          $drive->{bps_wr_max_length} || 1,
5425                          $drive->{iops_max_length} || 1,
5426                          $drive->{iops_rd_max_length} || 1,
5427                          $drive->{iops_wr_max_length} || 1,
5428                     );
5429
5430                 }
5431
5432                 return 1;
5433             }
5434
5435         } else { # cdrom
5436
5437             if ($drive->{file} eq 'none') {
5438                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5439                 if (drive_is_cloudinit($old_drive)) {
5440                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5441                 }
5442             } else {
5443                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5444
5445                 # force eject if locked
5446                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5447
5448                 if ($path) {
5449                     mon_cmd($vmid, "blockdev-change-medium",
5450                         id => "$opt", filename => "$path");
5451                 }
5452             }
5453
5454             return 1;
5455         }
5456     }
5457
5458     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5459     # hotplug new disks
5460     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5461     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5462 }
5463
5464 sub vmconfig_update_cloudinit_drive {
5465     my ($storecfg, $conf, $vmid) = @_;
5466
5467     my $cloudinit_ds = undef;
5468     my $cloudinit_drive = undef;
5469
5470     PVE::QemuConfig->foreach_volume($conf, sub {
5471         my ($ds, $drive) = @_;
5472         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5473             $cloudinit_ds = $ds;
5474             $cloudinit_drive = $drive;
5475         }
5476     });
5477
5478     return if !$cloudinit_drive;
5479
5480     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5481         PVE::QemuConfig->write_config($vmid, $conf);
5482     }
5483
5484     my $running = PVE::QemuServer::check_running($vmid);
5485
5486     if ($running) {
5487         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5488         if ($path) {
5489             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5490             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5491         }
5492     }
5493 }
5494
5495 # called in locked context by incoming migration
5496 sub vm_migrate_get_nbd_disks {
5497     my ($storecfg, $conf, $replicated_volumes) = @_;
5498
5499     my $local_volumes = {};
5500     PVE::QemuConfig->foreach_volume($conf, sub {
5501         my ($ds, $drive) = @_;
5502
5503         return if drive_is_cdrom($drive);
5504         return if $ds eq 'tpmstate0';
5505
5506         my $volid = $drive->{file};
5507
5508         return if !$volid;
5509
5510         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5511
5512         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5513         return if $scfg->{shared};
5514
5515         my $format = qemu_img_format($scfg, $volname);
5516
5517         # replicated disks re-use existing state via bitmap
5518         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5519         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5520     });
5521     return $local_volumes;
5522 }
5523
5524 # called in locked context by incoming migration
5525 sub vm_migrate_alloc_nbd_disks {
5526     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5527
5528     my $nbd = {};
5529     foreach my $opt (sort keys %$source_volumes) {
5530         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5531
5532         if ($use_existing) {
5533             $nbd->{$opt}->{drivestr} = print_drive($drive);
5534             $nbd->{$opt}->{volid} = $volid;
5535             $nbd->{$opt}->{replicated} = 1;
5536             next;
5537         }
5538
5539         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5540
5541         # order of precedence, filtered by whether storage supports it:
5542         # 1. explicit requested format
5543         # 2. default format of storage
5544         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5545         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5546
5547         my $size = $drive->{size} / 1024;
5548         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5549         my $newdrive = $drive;
5550         $newdrive->{format} = $format;
5551         $newdrive->{file} = $newvolid;
5552         my $drivestr = print_drive($newdrive);
5553         $nbd->{$opt}->{drivestr} = $drivestr;
5554         $nbd->{$opt}->{volid} = $newvolid;
5555     }
5556
5557     return $nbd;
5558 }
5559
5560 # see vm_start_nolock for parameters, additionally:
5561 # migrate_opts:
5562 #   storagemap = parsed storage map for allocating NBD disks
5563 sub vm_start {
5564     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5565
5566     return PVE::QemuConfig->lock_config($vmid, sub {
5567         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5568
5569         die "you can't start a vm if it's a template\n"
5570             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5571
5572         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5573         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5574
5575         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5576
5577         if ($has_backup_lock && $running) {
5578             # a backup is currently running, attempt to start the guest in the
5579             # existing QEMU instance
5580             return vm_resume($vmid);
5581         }
5582
5583         PVE::QemuConfig->check_lock($conf)
5584             if !($params->{skiplock} || $has_suspended_lock);
5585
5586         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5587
5588         die "VM $vmid already running\n" if $running;
5589
5590         if (my $storagemap = $migrate_opts->{storagemap}) {
5591             my $replicated = $migrate_opts->{replicated_volumes};
5592             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5593             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5594
5595             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5596                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5597             }
5598         }
5599
5600         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5601     });
5602 }
5603
5604
5605 # params:
5606 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5607 #   skiplock => 0/1, skip checking for config lock
5608 #   skiptemplate => 0/1, skip checking whether VM is template
5609 #   forcemachine => to force QEMU machine (rollback/migration)
5610 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5611 #   timeout => in seconds
5612 #   paused => start VM in paused state (backup)
5613 #   resume => resume from hibernation
5614 #   pbs-backing => {
5615 #      sata0 => {
5616 #         repository
5617 #         snapshot
5618 #         keyfile
5619 #         archive
5620 #      },
5621 #      virtio2 => ...
5622 #   }
5623 # migrate_opts:
5624 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5625 #   migratedfrom => source node
5626 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5627 #   network => CIDR of migration network
5628 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5629 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5630 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5631 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5632 #       contained in config
5633 sub vm_start_nolock {
5634     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5635
5636     my $statefile = $params->{statefile};
5637     my $resume = $params->{resume};
5638
5639     my $migratedfrom = $migrate_opts->{migratedfrom};
5640     my $migration_type = $migrate_opts->{type};
5641
5642     my $res = {};
5643
5644     # clean up leftover reboot request files
5645     eval { clear_reboot_request($vmid); };
5646     warn $@ if $@;
5647
5648     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5649         vmconfig_apply_pending($vmid, $conf, $storecfg);
5650         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5651     }
5652
5653     # don't regenerate the ISO if the VM is started as part of a live migration
5654     # this way we can reuse the old ISO with the correct config
5655     if (!$migratedfrom) {
5656         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5657             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5658             # $conf->{cloudinit}, so we could just not do this?
5659             # But we do it above, so for now let's be consistent.
5660             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5661         }
5662     }
5663
5664     # override offline migrated volumes, conf is out of date still
5665     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5666         for my $key (sort keys $offline_volumes->%*) {
5667             my $parsed = parse_drive($key, $conf->{$key});
5668             $parsed->{file} = $offline_volumes->{$key};
5669             $conf->{$key} = print_drive($parsed);
5670         }
5671     }
5672
5673     my $defaults = load_defaults();
5674
5675     # set environment variable useful inside network script
5676     # for remote migration the config is available on the target node!
5677     if (!$migrate_opts->{remote_node}) {
5678         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5679     }
5680
5681     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5682
5683     my $forcemachine = $params->{forcemachine};
5684     my $forcecpu = $params->{forcecpu};
5685     if ($resume) {
5686         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5687         $forcemachine = $conf->{runningmachine};
5688         $forcecpu = $conf->{runningcpu};
5689         print "Resuming suspended VM\n";
5690     }
5691
5692     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5693         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5694
5695     my $migration_ip;
5696     my $get_migration_ip = sub {
5697         my ($nodename) = @_;
5698
5699         return $migration_ip if defined($migration_ip);
5700
5701         my $cidr = $migrate_opts->{network};
5702
5703         if (!defined($cidr)) {
5704             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5705             $cidr = $dc_conf->{migration}->{network};
5706         }
5707
5708         if (defined($cidr)) {
5709             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5710
5711             die "could not get IP: no address configured on local " .
5712                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5713
5714             die "could not get IP: multiple addresses configured on local " .
5715                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5716
5717             $migration_ip = @$ips[0];
5718         }
5719
5720         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5721             if !defined($migration_ip);
5722
5723         return $migration_ip;
5724     };
5725
5726     if ($statefile) {
5727         if ($statefile eq 'tcp') {
5728             my $migrate = $res->{migrate} = { proto => 'tcp' };
5729             $migrate->{addr} = "localhost";
5730             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5731             my $nodename = nodename();
5732
5733             if (!defined($migration_type)) {
5734                 if (defined($datacenterconf->{migration}->{type})) {
5735                     $migration_type = $datacenterconf->{migration}->{type};
5736                 } else {
5737                     $migration_type = 'secure';
5738                 }
5739             }
5740
5741             if ($migration_type eq 'insecure') {
5742                 $migrate->{addr} = $get_migration_ip->($nodename);
5743                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5744             }
5745
5746             # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5747             # via QMP later
5748             push @$cmd, '-incoming', 'defer';
5749             push @$cmd, '-S';
5750
5751         } elsif ($statefile eq 'unix') {
5752             # should be default for secure migrations as a ssh TCP forward
5753             # tunnel is not deterministic reliable ready and fails regurarly
5754             # to set up in time, so use UNIX socket forwards
5755             my $migrate = $res->{migrate} = { proto => 'unix' };
5756             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5757             unlink $migrate->{addr};
5758
5759             $migrate->{uri} = "unix:$migrate->{addr}";
5760             push @$cmd, '-incoming', $migrate->{uri};
5761             push @$cmd, '-S';
5762
5763         } elsif (-e $statefile) {
5764             push @$cmd, '-loadstate', $statefile;
5765         } else {
5766             my $statepath = PVE::Storage::path($storecfg, $statefile);
5767             push @$vollist, $statefile;
5768             push @$cmd, '-loadstate', $statepath;
5769         }
5770     } elsif ($params->{paused}) {
5771         push @$cmd, '-S';
5772     }
5773
5774     my $memory = get_current_memory($conf->{memory});
5775     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5776
5777     my $pci_reserve_list = [];
5778     for my $device (values $pci_devices->%*) {
5779         next if $device->{mdev}; # we don't reserve for mdev devices
5780         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5781     }
5782
5783     # reserve all PCI IDs before actually doing anything with them
5784     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5785
5786     eval {
5787         my $uuid;
5788         for my $id (sort keys %$pci_devices) {
5789             my $d = $pci_devices->{$id};
5790             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5791
5792             my $chosen_mdev;
5793             for my $dev ($d->{ids}->@*) {
5794                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5795                 if ($d->{mdev}) {
5796                     warn $@ if $@;
5797                     $chosen_mdev = $info;
5798                     last if $chosen_mdev; # if successful, we're done
5799                 } else {
5800                     die $@ if $@;
5801                 }
5802             }
5803
5804             next if !$d->{mdev};
5805             die "could not create mediated device\n" if !defined($chosen_mdev);
5806
5807             # nvidia grid needs the uuid of the mdev as qemu parameter
5808             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5809                 if (defined($conf->{smbios1})) {
5810                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5811                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5812                 }
5813                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5814             }
5815         }
5816         push @$cmd, '-uuid', $uuid if defined($uuid);
5817     };
5818     if (my $err = $@) {
5819         eval { cleanup_pci_devices($vmid, $conf) };
5820         warn $@ if $@;
5821         die $err;
5822     }
5823
5824     PVE::Storage::activate_volumes($storecfg, $vollist);
5825
5826
5827     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5828     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5829     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5830     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5831     # timeout should be more than enough here...
5832     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5833
5834     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5835
5836     my %run_params = (
5837         timeout => $statefile ? undef : $start_timeout,
5838         umask => 0077,
5839         noerr => 1,
5840     );
5841
5842     # when migrating, prefix QEMU output so other side can pick up any
5843     # errors that might occur and show the user
5844     if ($migratedfrom) {
5845         $run_params{quiet} = 1;
5846         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5847     }
5848
5849     my %systemd_properties = (
5850         Slice => 'qemu.slice',
5851         KillMode => 'process',
5852         SendSIGKILL => 0,
5853         TimeoutStopUSec => ULONG_MAX, # infinity
5854     );
5855
5856     if (PVE::CGroup::cgroup_mode() == 2) {
5857         $systemd_properties{CPUWeight} = $cpuunits;
5858     } else {
5859         $systemd_properties{CPUShares} = $cpuunits;
5860     }
5861
5862     if (my $cpulimit = $conf->{cpulimit}) {
5863         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5864     }
5865     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5866
5867     my $run_qemu = sub {
5868         PVE::Tools::run_fork sub {
5869             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5870
5871             my $tpmpid;
5872             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5873                 # start the TPM emulator so QEMU can connect on start
5874                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5875             }
5876
5877             my $exitcode = run_command($cmd, %run_params);
5878             if ($exitcode) {
5879                 if ($tpmpid) {
5880                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5881                     kill 'TERM', $tpmpid;
5882                 }
5883                 die "QEMU exited with code $exitcode\n";
5884             }
5885         };
5886     };
5887
5888     if ($conf->{hugepages}) {
5889
5890         my $code = sub {
5891             my $hotplug_features =
5892                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5893             my $hugepages_topology =
5894                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5895
5896             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5897
5898             PVE::QemuServer::Memory::hugepages_mount();
5899             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5900
5901             eval { $run_qemu->() };
5902             if (my $err = $@) {
5903                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5904                     if !$conf->{keephugepages};
5905                 die $err;
5906             }
5907
5908             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5909                 if !$conf->{keephugepages};
5910         };
5911         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5912
5913     } else {
5914         eval { $run_qemu->() };
5915     }
5916
5917     if (my $err = $@) {
5918         # deactivate volumes if start fails
5919         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5920         warn $@ if $@;
5921         eval { cleanup_pci_devices($vmid, $conf) };
5922         warn $@ if $@;
5923
5924         die "start failed: $err";
5925     }
5926
5927     # re-reserve all PCI IDs now that we can know the actual VM PID
5928     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5929     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5930     warn $@ if $@;
5931
5932     if (defined(my $migrate = $res->{migrate})) {
5933         if ($migrate->{proto} eq 'tcp') {
5934             my $nodename = nodename();
5935             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5936             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5937             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5938             mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
5939         }
5940         print "migration listens on $migrate->{uri}\n";
5941     } elsif ($statefile) {
5942         eval { mon_cmd($vmid, "cont"); };
5943         warn $@ if $@;
5944     }
5945
5946     #start nbd server for storage migration
5947     if (my $nbd = $migrate_opts->{nbd}) {
5948         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5949
5950         my $migrate_storage_uri;
5951         # nbd_protocol_version > 0 for unix socket support
5952         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5953             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5954             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5955             $migrate_storage_uri = "nbd:unix:$socket_path";
5956             $res->{migrate}->{unix_sockets} = [$socket_path];
5957         } else {
5958             my $nodename = nodename();
5959             my $localip = $get_migration_ip->($nodename);
5960             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5961             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5962
5963             mon_cmd($vmid, "nbd-server-start", addr => {
5964                 type => 'inet',
5965                 data => {
5966                     host => "${localip}",
5967                     port => "${storage_migrate_port}",
5968                 },
5969             });
5970             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5971             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5972         }
5973
5974         my $block_info = mon_cmd($vmid, "query-block");
5975         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5976
5977         foreach my $opt (sort keys %$nbd) {
5978             my $drivestr = $nbd->{$opt}->{drivestr};
5979             my $volid = $nbd->{$opt}->{volid};
5980
5981             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5982
5983             mon_cmd(
5984                 $vmid,
5985                 "block-export-add",
5986                 id => "drive-$opt",
5987                 'node-name' => $block_node,
5988                 writable => JSON::true,
5989                 type => "nbd",
5990                 name => "drive-$opt", # NBD export name
5991             );
5992
5993             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5994             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5995             print "re-using replicated volume: $opt - $volid\n"
5996                 if $nbd->{$opt}->{replicated};
5997
5998             $res->{drives}->{$opt} = $nbd->{$opt};
5999             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6000         }
6001     }
6002
6003     if ($migratedfrom) {
6004         eval {
6005             set_migration_caps($vmid);
6006         };
6007         warn $@ if $@;
6008
6009         if ($spice_port) {
6010             print "spice listens on port $spice_port\n";
6011             $res->{spice_port} = $spice_port;
6012             if ($migrate_opts->{spice_ticket}) {
6013                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6014                     $migrate_opts->{spice_ticket});
6015                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6016             }
6017         }
6018
6019     } else {
6020         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6021             if !$statefile && $conf->{balloon};
6022
6023         foreach my $opt (keys %$conf) {
6024             next if $opt !~  m/^net\d+$/;
6025             my $nicconf = parse_net($conf->{$opt});
6026             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6027         }
6028         add_nets_bridge_fdb($conf, $vmid);
6029     }
6030
6031    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6032         eval {
6033             mon_cmd(
6034                 $vmid,
6035                 'qom-set',
6036                 path => "machine/peripheral/balloon0",
6037                 property => "guest-stats-polling-interval",
6038                 value => 2
6039             );
6040         };
6041         log_warn("could not set polling interval for ballooning - $@") if $@;
6042     }
6043
6044     if ($resume) {
6045         print "Resumed VM, removing state\n";
6046         if (my $vmstate = $conf->{vmstate}) {
6047             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6048             PVE::Storage::vdisk_free($storecfg, $vmstate);
6049         }
6050         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6051         PVE::QemuConfig->write_config($vmid, $conf);
6052     }
6053
6054     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6055
6056     my ($current_machine, $is_deprecated) =
6057         PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6058     if ($is_deprecated) {
6059         log_warn(
6060             "current machine version '$current_machine' is deprecated - see the documentation and ".
6061             "change to a newer one",
6062         );
6063     }
6064
6065     return $res;
6066 }
6067
6068 sub vm_commandline {
6069     my ($storecfg, $vmid, $snapname) = @_;
6070
6071     my $conf = PVE::QemuConfig->load_config($vmid);
6072
6073     my ($forcemachine, $forcecpu);
6074     if ($snapname) {
6075         my $snapshot = $conf->{snapshots}->{$snapname};
6076         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6077
6078         # check for machine or CPU overrides in snapshot
6079         $forcemachine = $snapshot->{runningmachine};
6080         $forcecpu = $snapshot->{runningcpu};
6081
6082         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6083
6084         $conf = $snapshot;
6085     }
6086
6087     my $defaults = load_defaults();
6088
6089     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6090
6091     return PVE::Tools::cmd2string($cmd);
6092 }
6093
6094 sub vm_reset {
6095     my ($vmid, $skiplock) = @_;
6096
6097     PVE::QemuConfig->lock_config($vmid, sub {
6098
6099         my $conf = PVE::QemuConfig->load_config($vmid);
6100
6101         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6102
6103         mon_cmd($vmid, "system_reset");
6104     });
6105 }
6106
6107 sub get_vm_volumes {
6108     my ($conf) = @_;
6109
6110     my $vollist = [];
6111     foreach_volid($conf, sub {
6112         my ($volid, $attr) = @_;
6113
6114         return if $volid =~ m|^/|;
6115
6116         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6117         return if !$sid;
6118
6119         push @$vollist, $volid;
6120     });
6121
6122     return $vollist;
6123 }
6124
6125 sub cleanup_pci_devices {
6126     my ($vmid, $conf) = @_;
6127
6128     foreach my $key (keys %$conf) {
6129         next if $key !~ m/^hostpci(\d+)$/;
6130         my $hostpciindex = $1;
6131         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6132         my $d = parse_hostpci($conf->{$key});
6133         if ($d->{mdev}) {
6134             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6135             # don't want to break ABI just for this two liner
6136             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6137
6138             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6139             # out when we do it first. so wait for up to 10 seconds and then try it manually
6140             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/ && -e $dev_sysfs_dir) {
6141                 my $count = 0;
6142                 while (-e $dev_sysfs_dir && $count < 10) {
6143                     sleep 1;
6144                     $count++;
6145                 }
6146                 print "waited $count seconds for mediated device driver finishing clean up\n";
6147             }
6148
6149             if (-e $dev_sysfs_dir) {
6150                 print "actively clean up mediated device with UUID $uuid\n";
6151                 PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1");
6152             }
6153         }
6154     }
6155     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6156 }
6157
6158 sub vm_stop_cleanup {
6159     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6160
6161     eval {
6162
6163         if (!$keepActive) {
6164             my $vollist = get_vm_volumes($conf);
6165             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6166
6167             if (my $tpmdrive = $conf->{tpmstate0}) {
6168                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6169                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6170                 if ($storeid) {
6171                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6172                 }
6173             }
6174         }
6175
6176         foreach my $ext (qw(mon qmp pid vnc qga)) {
6177             unlink "/var/run/qemu-server/${vmid}.$ext";
6178         }
6179
6180         if ($conf->{ivshmem}) {
6181             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6182             # just delete it for now, VMs which have this already open do not
6183             # are affected, but new VMs will get a separated one. If this
6184             # becomes an issue we either add some sort of ref-counting or just
6185             # add a "don't delete on stop" flag to the ivshmem format.
6186             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6187         }
6188
6189         cleanup_pci_devices($vmid, $conf);
6190
6191         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6192     };
6193     warn $@ if $@; # avoid errors - just warn
6194 }
6195
6196 # call only in locked context
6197 sub _do_vm_stop {
6198     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6199
6200     my $pid = check_running($vmid, $nocheck);
6201     return if !$pid;
6202
6203     my $conf;
6204     if (!$nocheck) {
6205         $conf = PVE::QemuConfig->load_config($vmid);
6206         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6207         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6208             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6209             $timeout = $opts->{down} if $opts->{down};
6210         }
6211         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6212     }
6213
6214     eval {
6215         if ($shutdown) {
6216             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6217                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6218             } else {
6219                 mon_cmd($vmid, "system_powerdown");
6220             }
6221         } else {
6222             mon_cmd($vmid, "quit");
6223         }
6224     };
6225     my $err = $@;
6226
6227     if (!$err) {
6228         $timeout = 60 if !defined($timeout);
6229
6230         my $count = 0;
6231         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6232             $count++;
6233             sleep 1;
6234         }
6235
6236         if ($count >= $timeout) {
6237             if ($force) {
6238                 warn "VM still running - terminating now with SIGTERM\n";
6239                 kill 15, $pid;
6240             } else {
6241                 die "VM quit/powerdown failed - got timeout\n";
6242             }
6243         } else {
6244             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6245             return;
6246         }
6247     } else {
6248        if (!check_running($vmid, $nocheck)) {
6249             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6250             return;
6251        }
6252        if ($force) {
6253             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6254             kill 15, $pid;
6255         } else {
6256             die "VM quit/powerdown failed\n";
6257         }
6258     }
6259
6260     # wait again
6261     $timeout = 10;
6262
6263     my $count = 0;
6264     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6265         $count++;
6266         sleep 1;
6267     }
6268
6269     if ($count >= $timeout) {
6270         warn "VM still running - terminating now with SIGKILL\n";
6271         kill 9, $pid;
6272         sleep 1;
6273     }
6274
6275     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6276 }
6277
6278 # Note: use $nocheck to skip tests if VM configuration file exists.
6279 # We need that when migration VMs to other nodes (files already moved)
6280 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6281 sub vm_stop {
6282     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6283
6284     $force = 1 if !defined($force) && !$shutdown;
6285
6286     if ($migratedfrom){
6287         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6288         kill 15, $pid if $pid;
6289         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6290         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6291         return;
6292     }
6293
6294     PVE::QemuConfig->lock_config($vmid, sub {
6295         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6296    });
6297 }
6298
6299 sub vm_reboot {
6300     my ($vmid, $timeout) = @_;
6301
6302     PVE::QemuConfig->lock_config($vmid, sub {
6303         eval {
6304
6305             # only reboot if running, as qmeventd starts it again on a stop event
6306             return if !check_running($vmid);
6307
6308             create_reboot_request($vmid);
6309
6310             my $storecfg = PVE::Storage::config();
6311             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6312
6313         };
6314         if (my $err = $@) {
6315             # avoid that the next normal shutdown will be confused for a reboot
6316             clear_reboot_request($vmid);
6317             die $err;
6318         }
6319    });
6320 }
6321
6322 # note: if using the statestorage parameter, the caller has to check privileges
6323 sub vm_suspend {
6324     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6325
6326     my $conf;
6327     my $path;
6328     my $storecfg;
6329     my $vmstate;
6330
6331     PVE::QemuConfig->lock_config($vmid, sub {
6332
6333         $conf = PVE::QemuConfig->load_config($vmid);
6334
6335         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6336         PVE::QemuConfig->check_lock($conf)
6337             if !($skiplock || $is_backing_up);
6338
6339         die "cannot suspend to disk during backup\n"
6340             if $is_backing_up && $includestate;
6341
6342         if ($includestate) {
6343             $conf->{lock} = 'suspending';
6344             my $date = strftime("%Y-%m-%d", localtime(time()));
6345             $storecfg = PVE::Storage::config();
6346             if (!$statestorage) {
6347                 $statestorage = find_vmstate_storage($conf, $storecfg);
6348                 # check permissions for the storage
6349                 my $rpcenv = PVE::RPCEnvironment::get();
6350                 if ($rpcenv->{type} ne 'cli') {
6351                     my $authuser = $rpcenv->get_user();
6352                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6353                 }
6354             }
6355
6356
6357             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6358                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6359             $path = PVE::Storage::path($storecfg, $vmstate);
6360             PVE::QemuConfig->write_config($vmid, $conf);
6361         } else {
6362             mon_cmd($vmid, "stop");
6363         }
6364     });
6365
6366     if ($includestate) {
6367         # save vm state
6368         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6369
6370         eval {
6371             set_migration_caps($vmid, 1);
6372             mon_cmd($vmid, "savevm-start", statefile => $path);
6373             for(;;) {
6374                 my $state = mon_cmd($vmid, "query-savevm");
6375                 if (!$state->{status}) {
6376                     die "savevm not active\n";
6377                 } elsif ($state->{status} eq 'active') {
6378                     sleep(1);
6379                     next;
6380                 } elsif ($state->{status} eq 'completed') {
6381                     print "State saved, quitting\n";
6382                     last;
6383                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6384                     die "query-savevm failed with error '$state->{error}'\n"
6385                 } else {
6386                     die "query-savevm returned status '$state->{status}'\n";
6387                 }
6388             }
6389         };
6390         my $err = $@;
6391
6392         PVE::QemuConfig->lock_config($vmid, sub {
6393             $conf = PVE::QemuConfig->load_config($vmid);
6394             if ($err) {
6395                 # cleanup, but leave suspending lock, to indicate something went wrong
6396                 eval {
6397                     mon_cmd($vmid, "savevm-end");
6398                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6399                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6400                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6401                     PVE::QemuConfig->write_config($vmid, $conf);
6402                 };
6403                 warn $@ if $@;
6404                 die $err;
6405             }
6406
6407             die "lock changed unexpectedly\n"
6408                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6409
6410             mon_cmd($vmid, "quit");
6411             $conf->{lock} = 'suspended';
6412             PVE::QemuConfig->write_config($vmid, $conf);
6413         });
6414     }
6415 }
6416
6417 # $nocheck is set when called as part of a migration - in this context the
6418 # location of the config file (source or target node) is not deterministic,
6419 # since migration cannot wait for pmxcfs to process the rename
6420 sub vm_resume {
6421     my ($vmid, $skiplock, $nocheck) = @_;
6422
6423     PVE::QemuConfig->lock_config($vmid, sub {
6424         my $res = mon_cmd($vmid, 'query-status');
6425         my $resume_cmd = 'cont';
6426         my $reset = 0;
6427         my $conf;
6428         if ($nocheck) {
6429             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6430             if ($@) {
6431                 my $vmlist = PVE::Cluster::get_vmlist();
6432                 if (exists($vmlist->{ids}->{$vmid})) {
6433                     my $node = $vmlist->{ids}->{$vmid}->{node};
6434                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6435                 }
6436                 if (!$conf) {
6437                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6438                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6439                 }
6440             }
6441         } else {
6442             $conf = PVE::QemuConfig->load_config($vmid);
6443         }
6444
6445         if ($res->{status}) {
6446             return if $res->{status} eq 'running'; # job done, go home
6447             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6448             $reset = 1 if $res->{status} eq 'shutdown';
6449         }
6450
6451         if (!$nocheck) {
6452             PVE::QemuConfig->check_lock($conf)
6453                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6454         }
6455
6456         if ($reset) {
6457             # required if a VM shuts down during a backup and we get a resume
6458             # request before the backup finishes for example
6459             mon_cmd($vmid, "system_reset");
6460         }
6461
6462         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6463
6464         mon_cmd($vmid, $resume_cmd);
6465     });
6466 }
6467
6468 sub vm_sendkey {
6469     my ($vmid, $skiplock, $key) = @_;
6470
6471     PVE::QemuConfig->lock_config($vmid, sub {
6472
6473         my $conf = PVE::QemuConfig->load_config($vmid);
6474
6475         # there is no qmp command, so we use the human monitor command
6476         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6477         die $res if $res ne '';
6478     });
6479 }
6480
6481 sub check_bridge_access {
6482     my ($rpcenv, $authuser, $conf) = @_;
6483
6484     return 1 if $authuser eq 'root@pam';
6485
6486     for my $opt (sort keys $conf->%*) {
6487         next if $opt !~ m/^net\d+$/;
6488         my $net = parse_net($conf->{$opt});
6489         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6490         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6491     }
6492     return 1;
6493 };
6494
6495 sub check_mapping_access {
6496     my ($rpcenv, $user, $conf) = @_;
6497
6498     for my $opt (keys $conf->%*) {
6499         if ($opt =~ m/^usb\d+$/) {
6500             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6501             if (my $host = $device->{host}) {
6502                 die "only root can set '$opt' config for real devices\n"
6503                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6504             } elsif ($device->{mapping}) {
6505                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6506             } else {
6507                 die "either 'host' or 'mapping' must be set.\n";
6508             }
6509         } elsif ($opt =~ m/^hostpci\d+$/) {
6510             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6511             if ($device->{host}) {
6512                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6513             } elsif ($device->{mapping}) {
6514                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6515             } else {
6516                 die "either 'host' or 'mapping' must be set.\n";
6517             }
6518        }
6519    }
6520 };
6521
6522 sub check_restore_permissions {
6523     my ($rpcenv, $user, $conf) = @_;
6524
6525     check_bridge_access($rpcenv, $user, $conf);
6526     check_mapping_access($rpcenv, $user, $conf);
6527 }
6528 # vzdump restore implementaion
6529
6530 sub tar_archive_read_firstfile {
6531     my $archive = shift;
6532
6533     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6534
6535     # try to detect archive type first
6536     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6537         die "unable to open file '$archive'\n";
6538     my $firstfile = <$fh>;
6539     kill 15, $pid;
6540     close $fh;
6541
6542     die "ERROR: archive contaions no data\n" if !$firstfile;
6543     chomp $firstfile;
6544
6545     return $firstfile;
6546 }
6547
6548 sub tar_restore_cleanup {
6549     my ($storecfg, $statfile) = @_;
6550
6551     print STDERR "starting cleanup\n";
6552
6553     if (my $fd = IO::File->new($statfile, "r")) {
6554         while (defined(my $line = <$fd>)) {
6555             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6556                 my $volid = $2;
6557                 eval {
6558                     if ($volid =~ m|^/|) {
6559                         unlink $volid || die 'unlink failed\n';
6560                     } else {
6561                         PVE::Storage::vdisk_free($storecfg, $volid);
6562                     }
6563                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6564                 };
6565                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6566             } else {
6567                 print STDERR "unable to parse line in statfile - $line";
6568             }
6569         }
6570         $fd->close();
6571     }
6572 }
6573
6574 sub restore_file_archive {
6575     my ($archive, $vmid, $user, $opts) = @_;
6576
6577     return restore_vma_archive($archive, $vmid, $user, $opts)
6578         if $archive eq '-';
6579
6580     my $info = PVE::Storage::archive_info($archive);
6581     my $format = $opts->{format} // $info->{format};
6582     my $comp = $info->{compression};
6583
6584     # try to detect archive format
6585     if ($format eq 'tar') {
6586         return restore_tar_archive($archive, $vmid, $user, $opts);
6587     } else {
6588         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6589     }
6590 }
6591
6592 # hepler to remove disks that will not be used after restore
6593 my $restore_cleanup_oldconf = sub {
6594     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6595
6596     my $kept_disks = {};
6597
6598     PVE::QemuConfig->foreach_volume($oldconf, sub {
6599         my ($ds, $drive) = @_;
6600
6601         return if drive_is_cdrom($drive, 1);
6602
6603         my $volid = $drive->{file};
6604         return if !$volid || $volid =~ m|^/|;
6605
6606         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6607         return if !$path || !$owner || ($owner != $vmid);
6608
6609         # Note: only delete disk we want to restore
6610         # other volumes will become unused
6611         if ($virtdev_hash->{$ds}) {
6612             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6613             if (my $err = $@) {
6614                 warn $err;
6615             }
6616         } else {
6617             $kept_disks->{$volid} = 1;
6618         }
6619     });
6620
6621     # after the restore we have no snapshots anymore
6622     for my $snapname (keys $oldconf->{snapshots}->%*) {
6623         my $snap = $oldconf->{snapshots}->{$snapname};
6624         if ($snap->{vmstate}) {
6625             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6626             if (my $err = $@) {
6627                 warn $err;
6628             }
6629         }
6630
6631         for my $volid (keys $kept_disks->%*) {
6632             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6633             warn $@ if $@;
6634         }
6635     }
6636 };
6637
6638 # Helper to parse vzdump backup device hints
6639 #
6640 # $rpcenv: Environment, used to ckeck storage permissions
6641 # $user: User ID, to check storage permissions
6642 # $storecfg: Storage configuration
6643 # $fh: the file handle for reading the configuration
6644 # $devinfo: should contain device sizes for all backu-up'ed devices
6645 # $options: backup options (pool, default storage)
6646 #
6647 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6648 my $parse_backup_hints = sub {
6649     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6650
6651     my $check_storage = sub { # assert if an image can be allocate
6652         my ($storeid, $scfg) = @_;
6653         die "Content type 'images' is not available on storage '$storeid'\n"
6654             if !$scfg->{content}->{images};
6655         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6656             if $user ne 'root@pam';
6657     };
6658
6659     my $virtdev_hash = {};
6660     while (defined(my $line = <$fh>)) {
6661         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6662             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6663             die "archive does not contain data for drive '$virtdev'\n"
6664                 if !$devinfo->{$devname};
6665
6666             if (defined($options->{storage})) {
6667                 $storeid = $options->{storage} || 'local';
6668             } elsif (!$storeid) {
6669                 $storeid = 'local';
6670             }
6671             $format = 'raw' if !$format;
6672             $devinfo->{$devname}->{devname} = $devname;
6673             $devinfo->{$devname}->{virtdev} = $virtdev;
6674             $devinfo->{$devname}->{format} = $format;
6675             $devinfo->{$devname}->{storeid} = $storeid;
6676
6677             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6678             $check_storage->($storeid, $scfg); # permission and content type check
6679
6680             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6681         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6682             my $virtdev = $1;
6683             my $drive = parse_drive($virtdev, $2);
6684
6685             if (drive_is_cloudinit($drive)) {
6686                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6687                 $storeid = $options->{storage} if defined ($options->{storage});
6688                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6689                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6690
6691                 $check_storage->($storeid, $scfg); # permission and content type check
6692
6693                 $virtdev_hash->{$virtdev} = {
6694                     format => $format,
6695                     storeid => $storeid,
6696                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6697                     is_cloudinit => 1,
6698                 };
6699             }
6700         }
6701     }
6702
6703     return $virtdev_hash;
6704 };
6705
6706 # Helper to allocate and activate all volumes required for a restore
6707 #
6708 # $storecfg: Storage configuration
6709 # $virtdev_hash: as returned by parse_backup_hints()
6710 #
6711 # Returns: { $virtdev => $volid }
6712 my $restore_allocate_devices = sub {
6713     my ($storecfg, $virtdev_hash, $vmid) = @_;
6714
6715     my $map = {};
6716     foreach my $virtdev (sort keys %$virtdev_hash) {
6717         my $d = $virtdev_hash->{$virtdev};
6718         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6719         my $storeid = $d->{storeid};
6720         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6721
6722         # test if requested format is supported
6723         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6724         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6725         $d->{format} = $defFormat if !$supported;
6726
6727         my $name;
6728         if ($d->{is_cloudinit}) {
6729             $name = "vm-$vmid-cloudinit";
6730             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6731             if ($scfg->{path}) {
6732                 $name .= ".$d->{format}";
6733             }
6734         }
6735
6736         my $volid = PVE::Storage::vdisk_alloc(
6737             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6738
6739         print STDERR "new volume ID is '$volid'\n";
6740         $d->{volid} = $volid;
6741
6742         PVE::Storage::activate_volumes($storecfg, [$volid]);
6743
6744         $map->{$virtdev} = $volid;
6745     }
6746
6747     return $map;
6748 };
6749
6750 sub restore_update_config_line {
6751     my ($cookie, $map, $line, $unique) = @_;
6752
6753     return '' if $line =~ m/^\#qmdump\#/;
6754     return '' if $line =~ m/^\#vzdump\#/;
6755     return '' if $line =~ m/^lock:/;
6756     return '' if $line =~ m/^unused\d+:/;
6757     return '' if $line =~ m/^parent:/;
6758
6759     my $res = '';
6760
6761     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6762     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6763         # try to convert old 1.X settings
6764         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6765         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6766             my ($model, $macaddr) = split(/\=/, $devconfig);
6767             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6768             my $net = {
6769                 model => $model,
6770                 bridge => "vmbr$ind",
6771                 macaddr => $macaddr,
6772             };
6773             my $netstr = print_net($net);
6774
6775             $res .= "net$cookie->{netcount}: $netstr\n";
6776             $cookie->{netcount}++;
6777         }
6778     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6779         my ($id, $netstr) = ($1, $2);
6780         my $net = parse_net($netstr);
6781         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6782         $netstr = print_net($net);
6783         $res .= "$id: $netstr\n";
6784     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6785         my $virtdev = $1;
6786         my $value = $3;
6787         my $di = parse_drive($virtdev, $value);
6788         if (defined($di->{backup}) && !$di->{backup}) {
6789             $res .= "#$line";
6790         } elsif ($map->{$virtdev}) {
6791             delete $di->{format}; # format can change on restore
6792             $di->{file} = $map->{$virtdev};
6793             $value = print_drive($di);
6794             $res .= "$virtdev: $value\n";
6795         } else {
6796             $res .= $line;
6797         }
6798     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6799         my $vmgenid = $1;
6800         if ($vmgenid ne '0') {
6801             # always generate a new vmgenid if there was a valid one setup
6802             $vmgenid = generate_uuid();
6803         }
6804         $res .= "vmgenid: $vmgenid\n";
6805     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6806         my ($uuid, $uuid_str);
6807         UUID::generate($uuid);
6808         UUID::unparse($uuid, $uuid_str);
6809         my $smbios1 = parse_smbios1($2);
6810         $smbios1->{uuid} = $uuid_str;
6811         $res .= $1.print_smbios1($smbios1)."\n";
6812     } else {
6813         $res .= $line;
6814     }
6815
6816     return $res;
6817 }
6818
6819 my $restore_deactivate_volumes = sub {
6820     my ($storecfg, $virtdev_hash) = @_;
6821
6822     my $vollist = [];
6823     for my $dev (values $virtdev_hash->%*) {
6824         push $vollist->@*, $dev->{volid} if $dev->{volid};
6825     }
6826
6827     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6828     print STDERR $@ if $@;
6829 };
6830
6831 my $restore_destroy_volumes = sub {
6832     my ($storecfg, $virtdev_hash) = @_;
6833
6834     for my $dev (values $virtdev_hash->%*) {
6835         my $volid = $dev->{volid} or next;
6836         eval {
6837             PVE::Storage::vdisk_free($storecfg, $volid);
6838             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6839         };
6840         print STDERR "unable to cleanup '$volid' - $@" if $@;
6841     }
6842 };
6843
6844 sub restore_merge_config {
6845     my ($filename, $backup_conf_raw, $override_conf) = @_;
6846
6847     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6848     for my $key (keys $override_conf->%*) {
6849         $backup_conf->{$key} = $override_conf->{$key};
6850     }
6851
6852     return $backup_conf;
6853 }
6854
6855 sub scan_volids {
6856     my ($cfg, $vmid) = @_;
6857
6858     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6859
6860     my $volid_hash = {};
6861     foreach my $storeid (keys %$info) {
6862         foreach my $item (@{$info->{$storeid}}) {
6863             next if !($item->{volid} && $item->{size});
6864             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6865             $volid_hash->{$item->{volid}} = $item;
6866         }
6867     }
6868
6869     return $volid_hash;
6870 }
6871
6872 sub update_disk_config {
6873     my ($vmid, $conf, $volid_hash) = @_;
6874
6875     my $changes;
6876     my $prefix = "VM $vmid";
6877
6878     # used and unused disks
6879     my $referenced = {};
6880
6881     # Note: it is allowed to define multiple storages with same path (alias), so
6882     # we need to check both 'volid' and real 'path' (two different volid can point
6883     # to the same path).
6884
6885     my $referencedpath = {};
6886
6887     # update size info
6888     PVE::QemuConfig->foreach_volume($conf, sub {
6889         my ($opt, $drive) = @_;
6890
6891         my $volid = $drive->{file};
6892         return if !$volid;
6893         my $volume = $volid_hash->{$volid};
6894
6895         # mark volid as "in-use" for next step
6896         $referenced->{$volid} = 1;
6897         if ($volume && (my $path = $volume->{path})) {
6898             $referencedpath->{$path} = 1;
6899         }
6900
6901         return if drive_is_cdrom($drive);
6902         return if !$volume;
6903
6904         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6905         if (defined($updated)) {
6906             $changes = 1;
6907             $conf->{$opt} = print_drive($updated);
6908             print "$prefix ($opt): $msg\n";
6909         }
6910     });
6911
6912     # remove 'unusedX' entry if volume is used
6913     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6914         my ($opt, $drive) = @_;
6915
6916         my $volid = $drive->{file};
6917         return if !$volid;
6918
6919         my $path;
6920         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6921         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6922             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6923             $changes = 1;
6924             delete $conf->{$opt};
6925         }
6926
6927         $referenced->{$volid} = 1;
6928         $referencedpath->{$path} = 1 if $path;
6929     });
6930
6931     foreach my $volid (sort keys %$volid_hash) {
6932         next if $volid =~ m/vm-$vmid-state-/;
6933         next if $referenced->{$volid};
6934         my $path = $volid_hash->{$volid}->{path};
6935         next if !$path; # just to be sure
6936         next if $referencedpath->{$path};
6937         $changes = 1;
6938         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6939         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6940         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6941     }
6942
6943     return $changes;
6944 }
6945
6946 sub rescan {
6947     my ($vmid, $nolock, $dryrun) = @_;
6948
6949     my $cfg = PVE::Storage::config();
6950
6951     print "rescan volumes...\n";
6952     my $volid_hash = scan_volids($cfg, $vmid);
6953
6954     my $updatefn =  sub {
6955         my ($vmid) = @_;
6956
6957         my $conf = PVE::QemuConfig->load_config($vmid);
6958
6959         PVE::QemuConfig->check_lock($conf);
6960
6961         my $vm_volids = {};
6962         foreach my $volid (keys %$volid_hash) {
6963             my $info = $volid_hash->{$volid};
6964             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6965         }
6966
6967         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6968
6969         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6970     };
6971
6972     if (defined($vmid)) {
6973         if ($nolock) {
6974             &$updatefn($vmid);
6975         } else {
6976             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6977         }
6978     } else {
6979         my $vmlist = config_list();
6980         foreach my $vmid (keys %$vmlist) {
6981             if ($nolock) {
6982                 &$updatefn($vmid);
6983             } else {
6984                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6985             }
6986         }
6987     }
6988 }
6989
6990 sub restore_proxmox_backup_archive {
6991     my ($archive, $vmid, $user, $options) = @_;
6992
6993     my $storecfg = PVE::Storage::config();
6994
6995     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6996     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6997
6998     my $fingerprint = $scfg->{fingerprint};
6999     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
7000
7001     my $repo = PVE::PBSClient::get_repository($scfg);
7002     my $namespace = $scfg->{namespace};
7003
7004     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7005     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7006     local $ENV{PBS_PASSWORD} = $password;
7007     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7008
7009     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7010         PVE::Storage::parse_volname($storecfg, $archive);
7011
7012     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7013
7014     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7015
7016     my $tmpdir = "/var/tmp/vzdumptmp$$";
7017     rmtree $tmpdir;
7018     mkpath $tmpdir;
7019
7020     my $conffile = PVE::QemuConfig->config_file($vmid);
7021      # disable interrupts (always do cleanups)
7022     local $SIG{INT} =
7023         local $SIG{TERM} =
7024         local $SIG{QUIT} =
7025         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7026
7027     # Note: $oldconf is undef if VM does not exists
7028     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7029     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7030     my $new_conf_raw = '';
7031
7032     my $rpcenv = PVE::RPCEnvironment::get();
7033     my $devinfo = {}; # info about drives included in backup
7034     my $virtdev_hash = {}; # info about allocated drives
7035
7036     eval {
7037         # enable interrupts
7038         local $SIG{INT} =
7039             local $SIG{TERM} =
7040             local $SIG{QUIT} =
7041             local $SIG{HUP} =
7042             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7043
7044         my $cfgfn = "$tmpdir/qemu-server.conf";
7045         my $firewall_config_fn = "$tmpdir/fw.conf";
7046         my $index_fn = "$tmpdir/index.json";
7047
7048         my $cmd = "restore";
7049
7050         my $param = [$pbs_backup_name, "index.json", $index_fn];
7051         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7052         my $index = PVE::Tools::file_get_contents($index_fn);
7053         $index = decode_json($index);
7054
7055         foreach my $info (@{$index->{files}}) {
7056             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7057                 my $devname = $1;
7058                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7059                     $devinfo->{$devname}->{size} = $1;
7060                 } else {
7061                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7062                 }
7063             }
7064         }
7065
7066         my $is_qemu_server_backup = scalar(
7067             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7068         );
7069         if (!$is_qemu_server_backup) {
7070             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7071         }
7072         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7073
7074         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7075         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7076
7077         if ($has_firewall_config) {
7078             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7079             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7080
7081             my $pve_firewall_dir = '/etc/pve/firewall';
7082             mkdir $pve_firewall_dir; # make sure the dir exists
7083             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7084         }
7085
7086         my $fh = IO::File->new($cfgfn, "r") ||
7087             die "unable to read qemu-server.conf - $!\n";
7088
7089         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7090
7091         # fixme: rate limit?
7092
7093         # create empty/temp config
7094         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7095
7096         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7097
7098         # allocate volumes
7099         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7100
7101         foreach my $virtdev (sort keys %$virtdev_hash) {
7102             my $d = $virtdev_hash->{$virtdev};
7103             next if $d->{is_cloudinit}; # no need to restore cloudinit
7104
7105             # this fails if storage is unavailable
7106             my $volid = $d->{volid};
7107             my $path = PVE::Storage::path($storecfg, $volid);
7108
7109             # for live-restore we only want to preload the efidisk and TPM state
7110             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7111
7112             my @ns_arg;
7113             if (defined(my $ns = $scfg->{namespace})) {
7114                 @ns_arg = ('--ns', $ns);
7115             }
7116
7117             my $pbs_restore_cmd = [
7118                 '/usr/bin/pbs-restore',
7119                 '--repository', $repo,
7120                 @ns_arg,
7121                 $pbs_backup_name,
7122                 "$d->{devname}.img.fidx",
7123                 $path,
7124                 '--verbose',
7125                 ];
7126
7127             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7128             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7129
7130             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7131                 push @$pbs_restore_cmd, '--skip-zero';
7132             }
7133
7134             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7135             print "restore proxmox backup image: $dbg_cmdstring\n";
7136             run_command($pbs_restore_cmd);
7137         }
7138
7139         $fh->seek(0, 0) || die "seek failed - $!\n";
7140
7141         my $cookie = { netcount => 0 };
7142         while (defined(my $line = <$fh>)) {
7143             $new_conf_raw .= restore_update_config_line(
7144                 $cookie,
7145                 $map,
7146                 $line,
7147                 $options->{unique},
7148             );
7149         }
7150
7151         $fh->close();
7152     };
7153     my $err = $@;
7154
7155     if ($err || !$options->{live}) {
7156         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7157     }
7158
7159     rmtree $tmpdir;
7160
7161     if ($err) {
7162         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7163         die $err;
7164     }
7165
7166     if ($options->{live}) {
7167         # keep lock during live-restore
7168         $new_conf_raw .= "\nlock: create";
7169     }
7170
7171     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7172     check_restore_permissions($rpcenv, $user, $new_conf);
7173     PVE::QemuConfig->write_config($vmid, $new_conf);
7174
7175     eval { rescan($vmid, 1); };
7176     warn $@ if $@;
7177
7178     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7179
7180     if ($options->{live}) {
7181         # enable interrupts
7182         local $SIG{INT} =
7183             local $SIG{TERM} =
7184             local $SIG{QUIT} =
7185             local $SIG{HUP} =
7186             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7187
7188         my $conf = PVE::QemuConfig->load_config($vmid);
7189         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7190
7191         # these special drives are already restored before start
7192         delete $devinfo->{'drive-efidisk0'};
7193         delete $devinfo->{'drive-tpmstate0-backup'};
7194
7195         my $pbs_opts = {
7196             repo => $repo,
7197             keyfile => $keyfile,
7198             snapshot => $pbs_backup_name,
7199             namespace => $namespace,
7200         };
7201         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7202
7203         PVE::QemuConfig->remove_lock($vmid, "create");
7204     }
7205 }
7206
7207 sub pbs_live_restore {
7208     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7209
7210     print "starting VM for live-restore\n";
7211     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7212
7213     my $pbs_backing = {};
7214     for my $ds (keys %$restored_disks) {
7215         $ds =~ m/^drive-(.*)$/;
7216         my $confname = $1;
7217         $pbs_backing->{$confname} = {
7218             repository => $opts->{repo},
7219             snapshot => $opts->{snapshot},
7220             archive => "$ds.img.fidx",
7221         };
7222         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7223         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7224
7225         my $drive = parse_drive($confname, $conf->{$confname});
7226         print "restoring '$ds' to '$drive->{file}'\n";
7227     }
7228
7229     my $drives_streamed = 0;
7230     eval {
7231         # make sure HA doesn't interrupt our restore by stopping the VM
7232         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7233             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7234         }
7235
7236         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7237         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7238         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7239
7240         my $qmeventd_fd = register_qmeventd_handle($vmid);
7241
7242         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7243         # this will effectively collapse the backing image chain consisting of
7244         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7245         # removes itself once all backing images vanish with 'auto-remove=on')
7246         my $jobs = {};
7247         for my $ds (sort keys %$restored_disks) {
7248             my $job_id = "restore-$ds";
7249             mon_cmd($vmid, 'block-stream',
7250                 'job-id' => $job_id,
7251                 device => "$ds",
7252             );
7253             $jobs->{$job_id} = {};
7254         }
7255
7256         mon_cmd($vmid, 'cont');
7257         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7258
7259         print "restore-drive jobs finished successfully, removing all tracking block devices"
7260             ." to disconnect from Proxmox Backup Server\n";
7261
7262         for my $ds (sort keys %$restored_disks) {
7263             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7264         }
7265
7266         close($qmeventd_fd);
7267     };
7268
7269     my $err = $@;
7270
7271     if ($err) {
7272         warn "An error occurred during live-restore: $err\n";
7273         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7274         die "live-restore failed\n";
7275     }
7276 }
7277
7278 sub restore_vma_archive {
7279     my ($archive, $vmid, $user, $opts, $comp) = @_;
7280
7281     my $readfrom = $archive;
7282
7283     my $cfg = PVE::Storage::config();
7284     my $commands = [];
7285     my $bwlimit = $opts->{bwlimit};
7286
7287     my $dbg_cmdstring = '';
7288     my $add_pipe = sub {
7289         my ($cmd) = @_;
7290         push @$commands, $cmd;
7291         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7292         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7293         $readfrom = '-';
7294     };
7295
7296     my $input = undef;
7297     if ($archive eq '-') {
7298         $input = '<&STDIN';
7299     } else {
7300         # If we use a backup from a PVE defined storage we also consider that
7301         # storage's rate limit:
7302         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7303         if (defined($volid)) {
7304             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7305             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7306             if ($readlimit) {
7307                 print STDERR "applying read rate limit: $readlimit\n";
7308                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7309                 $add_pipe->($cstream);
7310             }
7311         }
7312     }
7313
7314     if ($comp) {
7315         my $info = PVE::Storage::decompressor_info('vma', $comp);
7316         my $cmd = $info->{decompressor};
7317         push @$cmd, $readfrom;
7318         $add_pipe->($cmd);
7319     }
7320
7321     my $tmpdir = "/var/tmp/vzdumptmp$$";
7322     rmtree $tmpdir;
7323
7324     # disable interrupts (always do cleanups)
7325     local $SIG{INT} =
7326         local $SIG{TERM} =
7327         local $SIG{QUIT} =
7328         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7329
7330     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7331     POSIX::mkfifo($mapfifo, 0600);
7332     my $fifofh;
7333     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7334
7335     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7336
7337     my $devinfo = {}; # info about drives included in backup
7338     my $virtdev_hash = {}; # info about allocated drives
7339
7340     my $rpcenv = PVE::RPCEnvironment::get();
7341
7342     my $conffile = PVE::QemuConfig->config_file($vmid);
7343
7344     # Note: $oldconf is undef if VM does not exist
7345     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7346     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7347     my $new_conf_raw = '';
7348
7349     my %storage_limits;
7350
7351     my $print_devmap = sub {
7352         my $cfgfn = "$tmpdir/qemu-server.conf";
7353
7354         # we can read the config - that is already extracted
7355         my $fh = IO::File->new($cfgfn, "r") ||
7356             die "unable to read qemu-server.conf - $!\n";
7357
7358         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7359         if (-f $fwcfgfn) {
7360             my $pve_firewall_dir = '/etc/pve/firewall';
7361             mkdir $pve_firewall_dir; # make sure the dir exists
7362             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7363         }
7364
7365         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7366
7367         foreach my $info (values %{$virtdev_hash}) {
7368             my $storeid = $info->{storeid};
7369             next if defined($storage_limits{$storeid});
7370
7371             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7372             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7373             $storage_limits{$storeid} = $limit * 1024;
7374         }
7375
7376         foreach my $devname (keys %$devinfo) {
7377             die "found no device mapping information for device '$devname'\n"
7378                 if !$devinfo->{$devname}->{virtdev};
7379         }
7380
7381         # create empty/temp config
7382         if ($oldconf) {
7383             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7384             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7385         }
7386
7387         # allocate volumes
7388         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7389
7390         # print restore information to $fifofh
7391         foreach my $virtdev (sort keys %$virtdev_hash) {
7392             my $d = $virtdev_hash->{$virtdev};
7393             next if $d->{is_cloudinit}; # no need to restore cloudinit
7394
7395             my $storeid = $d->{storeid};
7396             my $volid = $d->{volid};
7397
7398             my $map_opts = '';
7399             if (my $limit = $storage_limits{$storeid}) {
7400                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7401             }
7402
7403             my $write_zeros = 1;
7404             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7405                 $write_zeros = 0;
7406             }
7407
7408             my $path = PVE::Storage::path($cfg, $volid);
7409
7410             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7411
7412             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7413         }
7414
7415         $fh->seek(0, 0) || die "seek failed - $!\n";
7416
7417         my $cookie = { netcount => 0 };
7418         while (defined(my $line = <$fh>)) {
7419             $new_conf_raw .= restore_update_config_line(
7420                 $cookie,
7421                 $map,
7422                 $line,
7423                 $opts->{unique},
7424             );
7425         }
7426
7427         $fh->close();
7428     };
7429
7430     my $oldtimeout;
7431
7432     eval {
7433         # enable interrupts
7434         local $SIG{INT} =
7435             local $SIG{TERM} =
7436             local $SIG{QUIT} =
7437             local $SIG{HUP} =
7438             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7439         local $SIG{ALRM} = sub { die "got timeout\n"; };
7440
7441         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7442
7443         my $parser = sub {
7444             my $line = shift;
7445
7446             print "$line\n";
7447
7448             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7449                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7450                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7451             } elsif ($line =~ m/^CTIME: /) {
7452                 # we correctly received the vma config, so we can disable
7453                 # the timeout now for disk allocation
7454                 alarm($oldtimeout || 0);
7455                 $oldtimeout = undef;
7456                 &$print_devmap();
7457                 print $fifofh "done\n";
7458                 close($fifofh);
7459                 $fifofh = undef;
7460             }
7461         };
7462
7463         print "restore vma archive: $dbg_cmdstring\n";
7464         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7465     };
7466     my $err = $@;
7467
7468     alarm($oldtimeout) if $oldtimeout;
7469
7470     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7471
7472     close($fifofh) if $fifofh;
7473     unlink $mapfifo;
7474     rmtree $tmpdir;
7475
7476     if ($err) {
7477         $restore_destroy_volumes->($cfg, $virtdev_hash);
7478         die $err;
7479     }
7480
7481     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7482     check_restore_permissions($rpcenv, $user, $new_conf);
7483     PVE::QemuConfig->write_config($vmid, $new_conf);
7484
7485     eval { rescan($vmid, 1); };
7486     warn $@ if $@;
7487
7488     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7489 }
7490
7491 sub restore_tar_archive {
7492     my ($archive, $vmid, $user, $opts) = @_;
7493
7494     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7495         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7496         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7497     }
7498
7499     if ($archive ne '-') {
7500         my $firstfile = tar_archive_read_firstfile($archive);
7501         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7502             if $firstfile ne 'qemu-server.conf';
7503     }
7504
7505     my $storecfg = PVE::Storage::config();
7506
7507     # avoid zombie disks when restoring over an existing VM -> cleanup first
7508     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7509     # skiplock=1 because qmrestore has set the 'create' lock itself already
7510     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7511     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7512
7513     my $tocmd = "/usr/lib/qemu-server/qmextract";
7514
7515     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7516     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7517     $tocmd .= ' --prealloc' if $opts->{prealloc};
7518     $tocmd .= ' --info' if $opts->{info};
7519
7520     # tar option "xf" does not autodetect compression when read from STDIN,
7521     # so we pipe to zcat
7522     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7523         PVE::Tools::shellquote("--to-command=$tocmd");
7524
7525     my $tmpdir = "/var/tmp/vzdumptmp$$";
7526     mkpath $tmpdir;
7527
7528     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7529     local $ENV{VZDUMP_VMID} = $vmid;
7530     local $ENV{VZDUMP_USER} = $user;
7531
7532     my $conffile = PVE::QemuConfig->config_file($vmid);
7533     my $new_conf_raw = '';
7534
7535     # disable interrupts (always do cleanups)
7536     local $SIG{INT} =
7537         local $SIG{TERM} =
7538         local $SIG{QUIT} =
7539         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7540
7541     eval {
7542         # enable interrupts
7543         local $SIG{INT} =
7544             local $SIG{TERM} =
7545             local $SIG{QUIT} =
7546             local $SIG{HUP} =
7547             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7548
7549         if ($archive eq '-') {
7550             print "extracting archive from STDIN\n";
7551             run_command($cmd, input => "<&STDIN");
7552         } else {
7553             print "extracting archive '$archive'\n";
7554             run_command($cmd);
7555         }
7556
7557         return if $opts->{info};
7558
7559         # read new mapping
7560         my $map = {};
7561         my $statfile = "$tmpdir/qmrestore.stat";
7562         if (my $fd = IO::File->new($statfile, "r")) {
7563             while (defined (my $line = <$fd>)) {
7564                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7565                     $map->{$1} = $2 if $1;
7566                 } else {
7567                     print STDERR "unable to parse line in statfile - $line\n";
7568                 }
7569             }
7570             $fd->close();
7571         }
7572
7573         my $confsrc = "$tmpdir/qemu-server.conf";
7574
7575         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7576
7577         my $cookie = { netcount => 0 };
7578         while (defined (my $line = <$srcfd>)) {
7579             $new_conf_raw .= restore_update_config_line(
7580                 $cookie,
7581                 $map,
7582                 $line,
7583                 $opts->{unique},
7584             );
7585         }
7586
7587         $srcfd->close();
7588     };
7589     if (my $err = $@) {
7590         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7591         die $err;
7592     }
7593
7594     rmtree $tmpdir;
7595
7596     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7597
7598     PVE::Cluster::cfs_update(); # make sure we read new file
7599
7600     eval { rescan($vmid, 1); };
7601     warn $@ if $@;
7602 };
7603
7604 sub foreach_storage_used_by_vm {
7605     my ($conf, $func) = @_;
7606
7607     my $sidhash = {};
7608
7609     PVE::QemuConfig->foreach_volume($conf, sub {
7610         my ($ds, $drive) = @_;
7611         return if drive_is_cdrom($drive);
7612
7613         my $volid = $drive->{file};
7614
7615         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7616         $sidhash->{$sid} = $sid if $sid;
7617     });
7618
7619     foreach my $sid (sort keys %$sidhash) {
7620         &$func($sid);
7621     }
7622 }
7623
7624 my $qemu_snap_storage = {
7625     rbd => 1,
7626 };
7627 sub do_snapshots_with_qemu {
7628     my ($storecfg, $volid, $deviceid) = @_;
7629
7630     return if $deviceid =~ m/tpmstate0/;
7631
7632     my $storage_name = PVE::Storage::parse_volume_id($volid);
7633     my $scfg = $storecfg->{ids}->{$storage_name};
7634     die "could not find storage '$storage_name'\n" if !defined($scfg);
7635
7636     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7637         return 1;
7638     }
7639
7640     if ($volid =~ m/\.(qcow2|qed)$/){
7641         return 1;
7642     }
7643
7644     return;
7645 }
7646
7647 sub qga_check_running {
7648     my ($vmid, $nowarn) = @_;
7649
7650     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7651     if ($@) {
7652         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7653         return 0;
7654     }
7655     return 1;
7656 }
7657
7658 sub template_create {
7659     my ($vmid, $conf, $disk) = @_;
7660
7661     my $storecfg = PVE::Storage::config();
7662
7663     PVE::QemuConfig->foreach_volume($conf, sub {
7664         my ($ds, $drive) = @_;
7665
7666         return if drive_is_cdrom($drive);
7667         return if $disk && $ds ne $disk;
7668
7669         my $volid = $drive->{file};
7670         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7671
7672         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7673         $drive->{file} = $voliddst;
7674         $conf->{$ds} = print_drive($drive);
7675         PVE::QemuConfig->write_config($vmid, $conf);
7676     });
7677 }
7678
7679 sub convert_iscsi_path {
7680     my ($path) = @_;
7681
7682     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7683         my $portal = $1;
7684         my $target = $2;
7685         my $lun = $3;
7686
7687         my $initiator_name = get_initiator_name();
7688
7689         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7690                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7691     }
7692
7693     die "cannot convert iscsi path '$path', unkown format\n";
7694 }
7695
7696 sub qemu_img_convert {
7697     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7698
7699     my $storecfg = PVE::Storage::config();
7700     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7701     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7702
7703     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7704
7705     my $cachemode;
7706     my $src_path;
7707     my $src_is_iscsi = 0;
7708     my $src_format;
7709
7710     if ($src_storeid) {
7711         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7712         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7713         $src_format = qemu_img_format($src_scfg, $src_volname);
7714         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7715         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7716         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7717     } elsif (-f $src_volid || -b $src_volid) {
7718         $src_path = $src_volid;
7719         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7720             $src_format = $1;
7721         }
7722     }
7723
7724     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7725
7726     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7727     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7728     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7729     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7730
7731     my $cmd = [];
7732     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7733     push @$cmd, '-l', "snapshot.name=$snapname"
7734         if $snapname && $src_format && $src_format eq "qcow2";
7735     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7736     push @$cmd, '-T', $cachemode if defined($cachemode);
7737     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7738
7739     if ($src_is_iscsi) {
7740         push @$cmd, '--image-opts';
7741         $src_path = convert_iscsi_path($src_path);
7742     } elsif ($src_format) {
7743         push @$cmd, '-f', $src_format;
7744     }
7745
7746     if ($dst_is_iscsi) {
7747         push @$cmd, '--target-image-opts';
7748         $dst_path = convert_iscsi_path($dst_path);
7749     } else {
7750         push @$cmd, '-O', $dst_format;
7751     }
7752
7753     push @$cmd, $src_path;
7754
7755     if (!$dst_is_iscsi && $is_zero_initialized) {
7756         push @$cmd, "zeroinit:$dst_path";
7757     } else {
7758         push @$cmd, $dst_path;
7759     }
7760
7761     my $parser = sub {
7762         my $line = shift;
7763         if($line =~ m/\((\S+)\/100\%\)/){
7764             my $percent = $1;
7765             my $transferred = int($size * $percent / 100);
7766             my $total_h = render_bytes($size, 1);
7767             my $transferred_h = render_bytes($transferred, 1);
7768
7769             print "transferred $transferred_h of $total_h ($percent%)\n";
7770         }
7771
7772     };
7773
7774     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7775     my $err = $@;
7776     die "copy failed: $err" if $err;
7777 }
7778
7779 sub qemu_img_format {
7780     my ($scfg, $volname) = @_;
7781
7782     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7783         return $1;
7784     } else {
7785         return "raw";
7786     }
7787 }
7788
7789 sub qemu_drive_mirror {
7790     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7791
7792     $jobs = {} if !$jobs;
7793
7794     my $qemu_target;
7795     my $format;
7796     $jobs->{"drive-$drive"} = {};
7797
7798     if ($dst_volid =~ /^nbd:/) {
7799         $qemu_target = $dst_volid;
7800         $format = "nbd";
7801     } else {
7802         my $storecfg = PVE::Storage::config();
7803         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7804
7805         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7806
7807         $format = qemu_img_format($dst_scfg, $dst_volname);
7808
7809         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7810
7811         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7812     }
7813
7814     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7815     $opts->{format} = $format if $format;
7816
7817     if (defined($src_bitmap)) {
7818         $opts->{sync} = 'incremental';
7819         $opts->{bitmap} = $src_bitmap;
7820         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7821     }
7822
7823     if (defined($bwlimit)) {
7824         $opts->{speed} = $bwlimit * 1024;
7825         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7826     } else {
7827         print "drive mirror is starting for drive-$drive\n";
7828     }
7829
7830     # if a job already runs for this device we get an error, catch it for cleanup
7831     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7832     if (my $err = $@) {
7833         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7834         warn "$@\n" if $@;
7835         die "mirroring error: $err\n";
7836     }
7837
7838     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7839 }
7840
7841 # $completion can be either
7842 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7843 # 'cancel': wait until all jobs are ready, block-job-cancel them
7844 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7845 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7846 sub qemu_drive_mirror_monitor {
7847     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7848
7849     $completion //= 'complete';
7850     $op //= "mirror";
7851
7852     eval {
7853         my $err_complete = 0;
7854
7855         my $starttime = time ();
7856         while (1) {
7857             die "block job ('$op') timed out\n" if $err_complete > 300;
7858
7859             my $stats = mon_cmd($vmid, "query-block-jobs");
7860             my $ctime = time();
7861
7862             my $running_jobs = {};
7863             for my $stat (@$stats) {
7864                 next if $stat->{type} ne $op;
7865                 $running_jobs->{$stat->{device}} = $stat;
7866             }
7867
7868             my $readycounter = 0;
7869
7870             for my $job_id (sort keys %$jobs) {
7871                 my $job = $running_jobs->{$job_id};
7872
7873                 my $vanished = !defined($job);
7874                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7875                 if($complete || ($vanished && $completion eq 'auto')) {
7876                     print "$job_id: $op-job finished\n";
7877                     delete $jobs->{$job_id};
7878                     next;
7879                 }
7880
7881                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7882
7883                 my $busy = $job->{busy};
7884                 my $ready = $job->{ready};
7885                 if (my $total = $job->{len}) {
7886                     my $transferred = $job->{offset} || 0;
7887                     my $remaining = $total - $transferred;
7888                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7889
7890                     my $duration = $ctime - $starttime;
7891                     my $total_h = render_bytes($total, 1);
7892                     my $transferred_h = render_bytes($transferred, 1);
7893
7894                     my $status = sprintf(
7895                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7896                         render_duration($duration),
7897                     );
7898
7899                     if ($ready) {
7900                         if ($busy) {
7901                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7902                         } else {
7903                             $status .= ", ready";
7904                         }
7905                     }
7906                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7907                     $jobs->{$job_id}->{ready} = $ready;
7908                 }
7909
7910                 $readycounter++ if $job->{ready};
7911             }
7912
7913             last if scalar(keys %$jobs) == 0;
7914
7915             if ($readycounter == scalar(keys %$jobs)) {
7916                 print "all '$op' jobs are ready\n";
7917
7918                 # do the complete later (or has already been done)
7919                 last if $completion eq 'skip' || $completion eq 'auto';
7920
7921                 if ($vmiddst && $vmiddst != $vmid) {
7922                     my $agent_running = $qga && qga_check_running($vmid);
7923                     if ($agent_running) {
7924                         print "freeze filesystem\n";
7925                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7926                         warn $@ if $@;
7927                     } else {
7928                         print "suspend vm\n";
7929                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7930                         warn $@ if $@;
7931                     }
7932
7933                     # if we clone a disk for a new target vm, we don't switch the disk
7934                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7935
7936                     if ($agent_running) {
7937                         print "unfreeze filesystem\n";
7938                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7939                         warn $@ if $@;
7940                     } else {
7941                         print "resume vm\n";
7942                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7943                         warn $@ if $@;
7944                     }
7945
7946                     last;
7947                 } else {
7948
7949                     for my $job_id (sort keys %$jobs) {
7950                         # try to switch the disk if source and destination are on the same guest
7951                         print "$job_id: Completing block job_id...\n";
7952
7953                         my $op;
7954                         if ($completion eq 'complete') {
7955                             $op = 'block-job-complete';
7956                         } elsif ($completion eq 'cancel') {
7957                             $op = 'block-job-cancel';
7958                         } else {
7959                             die "invalid completion value: $completion\n";
7960                         }
7961                         eval { mon_cmd($vmid, $op, device => $job_id) };
7962                         if ($@ =~ m/cannot be completed/) {
7963                             print "$job_id: block job cannot be completed, trying again.\n";
7964                             $err_complete++;
7965                         }else {
7966                             print "$job_id: Completed successfully.\n";
7967                             $jobs->{$job_id}->{complete} = 1;
7968                         }
7969                     }
7970                 }
7971             }
7972             sleep 1;
7973         }
7974     };
7975     my $err = $@;
7976
7977     if ($err) {
7978         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7979         die "block job ($op) error: $err";
7980     }
7981 }
7982
7983 sub qemu_blockjobs_cancel {
7984     my ($vmid, $jobs) = @_;
7985
7986     foreach my $job (keys %$jobs) {
7987         print "$job: Cancelling block job\n";
7988         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7989         $jobs->{$job}->{cancel} = 1;
7990     }
7991
7992     while (1) {
7993         my $stats = mon_cmd($vmid, "query-block-jobs");
7994
7995         my $running_jobs = {};
7996         foreach my $stat (@$stats) {
7997             $running_jobs->{$stat->{device}} = $stat;
7998         }
7999
8000         foreach my $job (keys %$jobs) {
8001
8002             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
8003                 print "$job: Done.\n";
8004                 delete $jobs->{$job};
8005             }
8006         }
8007
8008         last if scalar(keys %$jobs) == 0;
8009
8010         sleep 1;
8011     }
8012 }
8013
8014 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8015 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8016 my sub clone_disk_check_io_uring {
8017     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8018
8019     return if !$use_drive_mirror;
8020
8021     # Don't complain when not changing storage.
8022     # Assume if it works for the source, it'll work for the target too.
8023     return if $src_storeid eq $dst_storeid;
8024
8025     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8026     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8027
8028     my $cache_direct = drive_uses_cache_direct($src_drive);
8029
8030     my $src_uses_io_uring;
8031     if ($src_drive->{aio}) {
8032         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8033     } else {
8034         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8035     }
8036
8037     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8038         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8039 }
8040
8041 sub clone_disk {
8042     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8043
8044     my ($vmid, $running) = $source->@{qw(vmid running)};
8045     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8046
8047     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8048     my ($storage, $format) = $dest->@{qw(storage format)};
8049
8050     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8051
8052     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8053         die "cloning from/to EFI disk requires EFI disk\n"
8054             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8055         die "cloning from/to TPM state requires TPM state\n"
8056             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8057
8058         # This would lead to two device nodes in QEMU pointing to the same backing image!
8059         die "cannot change drive name when cloning disk from/to the same VM\n"
8060             if $use_drive_mirror && $vmid == $newvmid;
8061     }
8062
8063     die "cannot move TPM state while VM is running\n"
8064         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8065
8066     my $newvolid;
8067
8068     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8069     print "$src_drivename " if $src_drivename;
8070     print "($drive->{file})\n";
8071
8072     if (!$full) {
8073         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8074         push @$newvollist, $newvolid;
8075     } else {
8076         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8077         my $storeid = $storage || $src_storeid;
8078
8079         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8080
8081         my $name = undef;
8082         my $size = undef;
8083         if (drive_is_cloudinit($drive)) {
8084             $name = "vm-$newvmid-cloudinit";
8085             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8086             if ($scfg->{path}) {
8087                 $name .= ".$dst_format";
8088             }
8089             $snapname = undef;
8090             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8091         } elsif ($dst_drivename eq 'efidisk0') {
8092             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8093         } elsif ($dst_drivename eq 'tpmstate0') {
8094             $dst_format = 'raw';
8095             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8096         } else {
8097             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8098
8099             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8100         }
8101         $newvolid = PVE::Storage::vdisk_alloc(
8102             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8103         );
8104         push @$newvollist, $newvolid;
8105
8106         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8107
8108         if (drive_is_cloudinit($drive)) {
8109             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8110             # if this is the case, we have to complete any block-jobs still there from
8111             # previous drive-mirrors
8112             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8113                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8114             }
8115             goto no_data_clone;
8116         }
8117
8118         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8119         if ($use_drive_mirror) {
8120             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8121                 $completion, $qga, $bwlimit);
8122         } else {
8123             if ($dst_drivename eq 'efidisk0') {
8124                 # the relevant data on the efidisk may be smaller than the source
8125                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8126                 # that is given by the OVMF_VARS.fd
8127                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8128                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8129
8130                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8131
8132                 # better for Ceph if block size is not too small, see bug #3324
8133                 my $bs = 1024*1024;
8134
8135                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8136
8137                 if ($src_format eq 'qcow2' && $snapname) {
8138                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8139                         if !min_version(kvm_user_version(), 6, 2);
8140                     push $cmd->@*, '-l', $snapname;
8141                 }
8142                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8143                 run_command($cmd);
8144             } else {
8145                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8146             }
8147         }
8148     }
8149
8150 no_data_clone:
8151     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8152
8153     my $disk = dclone($drive);
8154     delete $disk->{format};
8155     $disk->{file} = $newvolid;
8156     $disk->{size} = $size if defined($size);
8157
8158     return $disk;
8159 }
8160
8161 sub get_running_qemu_version {
8162     my ($vmid) = @_;
8163     my $res = mon_cmd($vmid, "query-version");
8164     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8165 }
8166
8167 sub qemu_use_old_bios_files {
8168     my ($machine_type) = @_;
8169
8170     return if !$machine_type;
8171
8172     my $use_old_bios_files = undef;
8173
8174     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8175         $machine_type = $1;
8176         $use_old_bios_files = 1;
8177     } else {
8178         my $version = extract_version($machine_type, kvm_user_version());
8179         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8180         # load new efi bios files on migration. So this hack is required to allow
8181         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8182         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8183         $use_old_bios_files = !min_version($version, 2, 4);
8184     }
8185
8186     return ($use_old_bios_files, $machine_type);
8187 }
8188
8189 sub get_efivars_size {
8190     my ($conf, $efidisk) = @_;
8191
8192     my $arch = get_vm_arch($conf);
8193     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8194     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8195     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8196     return -s $ovmf_vars;
8197 }
8198
8199 sub update_efidisk_size {
8200     my ($conf) = @_;
8201
8202     return if !defined($conf->{efidisk0});
8203
8204     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8205     $disk->{size} = get_efivars_size($conf);
8206     $conf->{efidisk0} = print_drive($disk);
8207
8208     return;
8209 }
8210
8211 sub update_tpmstate_size {
8212     my ($conf) = @_;
8213
8214     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8215     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8216     $conf->{tpmstate0} = print_drive($disk);
8217 }
8218
8219 sub create_efidisk($$$$$$$) {
8220     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8221
8222     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8223
8224     my $vars_size_b = -s $ovmf_vars;
8225     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8226     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8227     PVE::Storage::activate_volumes($storecfg, [$volid]);
8228
8229     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8230     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8231
8232     return ($volid, $size/1024);
8233 }
8234
8235 sub vm_iothreads_list {
8236     my ($vmid) = @_;
8237
8238     my $res = mon_cmd($vmid, 'query-iothreads');
8239
8240     my $iothreads = {};
8241     foreach my $iothread (@$res) {
8242         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8243     }
8244
8245     return $iothreads;
8246 }
8247
8248 sub scsihw_infos {
8249     my ($conf, $drive) = @_;
8250
8251     my $maxdev = 0;
8252
8253     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8254         $maxdev = 7;
8255     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8256         $maxdev = 1;
8257     } else {
8258         $maxdev = 256;
8259     }
8260
8261     my $controller = int($drive->{index} / $maxdev);
8262     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8263         ? "virtioscsi"
8264         : "scsihw";
8265
8266     return ($maxdev, $controller, $controller_prefix);
8267 }
8268
8269 sub resolve_dst_disk_format {
8270         my ($storecfg, $storeid, $src_volname, $format) = @_;
8271         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8272
8273         if (!$format) {
8274             # if no target format is specified, use the source disk format as hint
8275             if ($src_volname) {
8276                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8277                 $format = qemu_img_format($scfg, $src_volname);
8278             } else {
8279                 return $defFormat;
8280             }
8281         }
8282
8283         # test if requested format is supported - else use default
8284         my $supported = grep { $_ eq $format } @$validFormats;
8285         $format = $defFormat if !$supported;
8286         return $format;
8287 }
8288
8289 # NOTE: if this logic changes, please update docs & possibly gui logic
8290 sub find_vmstate_storage {
8291     my ($conf, $storecfg) = @_;
8292
8293     # first, return storage from conf if set
8294     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8295
8296     my ($target, $shared, $local);
8297
8298     foreach_storage_used_by_vm($conf, sub {
8299         my ($sid) = @_;
8300         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8301         my $dst = $scfg->{shared} ? \$shared : \$local;
8302         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8303     });
8304
8305     # second, use shared storage where VM has at least one disk
8306     # third, use local storage where VM has at least one disk
8307     # fall back to local storage
8308     $target = $shared // $local // 'local';
8309
8310     return $target;
8311 }
8312
8313 sub generate_uuid {
8314     my ($uuid, $uuid_str);
8315     UUID::generate($uuid);
8316     UUID::unparse($uuid, $uuid_str);
8317     return $uuid_str;
8318 }
8319
8320 sub generate_smbios1_uuid {
8321     return "uuid=".generate_uuid();
8322 }
8323
8324 sub nbd_stop {
8325     my ($vmid) = @_;
8326
8327     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8328 }
8329
8330 sub create_reboot_request {
8331     my ($vmid) = @_;
8332     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8333         or die "failed to create reboot trigger file: $!\n";
8334     close($fh);
8335 }
8336
8337 sub clear_reboot_request {
8338     my ($vmid) = @_;
8339     my $path = "/run/qemu-server/$vmid.reboot";
8340     my $res = 0;
8341
8342     $res = unlink($path);
8343     die "could not remove reboot request for $vmid: $!"
8344         if !$res && $! != POSIX::ENOENT;
8345
8346     return $res;
8347 }
8348
8349 sub bootorder_from_legacy {
8350     my ($conf, $bootcfg) = @_;
8351
8352     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8353     my $bootindex_hash = {};
8354     my $i = 1;
8355     foreach my $o (split(//, $boot)) {
8356         $bootindex_hash->{$o} = $i*100;
8357         $i++;
8358     }
8359
8360     my $bootorder = {};
8361
8362     PVE::QemuConfig->foreach_volume($conf, sub {
8363         my ($ds, $drive) = @_;
8364
8365         if (drive_is_cdrom ($drive, 1)) {
8366             if ($bootindex_hash->{d}) {
8367                 $bootorder->{$ds} = $bootindex_hash->{d};
8368                 $bootindex_hash->{d} += 1;
8369             }
8370         } elsif ($bootindex_hash->{c}) {
8371             $bootorder->{$ds} = $bootindex_hash->{c}
8372                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8373             $bootindex_hash->{c} += 1;
8374         }
8375     });
8376
8377     if ($bootindex_hash->{n}) {
8378         for (my $i = 0; $i < $MAX_NETS; $i++) {
8379             my $netname = "net$i";
8380             next if !$conf->{$netname};
8381             $bootorder->{$netname} = $bootindex_hash->{n};
8382             $bootindex_hash->{n} += 1;
8383         }
8384     }
8385
8386     return $bootorder;
8387 }
8388
8389 # Generate default device list for 'boot: order=' property. Matches legacy
8390 # default boot order, but with explicit device names. This is important, since
8391 # the fallback for when neither 'order' nor the old format is specified relies
8392 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8393 sub get_default_bootdevices {
8394     my ($conf) = @_;
8395
8396     my @ret = ();
8397
8398     # harddisk
8399     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8400     push @ret, $first if $first;
8401
8402     # cdrom
8403     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8404     push @ret, $first if $first;
8405
8406     # network
8407     for (my $i = 0; $i < $MAX_NETS; $i++) {
8408         my $netname = "net$i";
8409         next if !$conf->{$netname};
8410         push @ret, $netname;
8411         last;
8412     }
8413
8414     return \@ret;
8415 }
8416
8417 sub device_bootorder {
8418     my ($conf) = @_;
8419
8420     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8421
8422     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8423
8424     my $bootorder = {};
8425     if (!defined($boot) || $boot->{legacy}) {
8426         $bootorder = bootorder_from_legacy($conf, $boot);
8427     } elsif ($boot->{order}) {
8428         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8429         for my $dev (PVE::Tools::split_list($boot->{order})) {
8430             $bootorder->{$dev} = $i++;
8431         }
8432     }
8433
8434     return $bootorder;
8435 }
8436
8437 sub register_qmeventd_handle {
8438     my ($vmid) = @_;
8439
8440     my $fh;
8441     my $peer = "/var/run/qmeventd.sock";
8442     my $count = 0;
8443
8444     for (;;) {
8445         $count++;
8446         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8447         last if $fh;
8448         if ($! != EINTR && $! != EAGAIN) {
8449             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8450         }
8451         if ($count > 4) {
8452             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8453               . "after $count retries\n";
8454         }
8455         usleep(25000);
8456     }
8457
8458     # send handshake to mark VM as backing up
8459     print $fh to_json({vzdump => {vmid => "$vmid"}});
8460
8461     # return handle to be closed later when inhibit is no longer required
8462     return $fh;
8463 }
8464
8465 # bash completion helper
8466
8467 sub complete_backup_archives {
8468     my ($cmdname, $pname, $cvalue) = @_;
8469
8470     my $cfg = PVE::Storage::config();
8471
8472     my $storeid;
8473
8474     if ($cvalue =~ m/^([^:]+):/) {
8475         $storeid = $1;
8476     }
8477
8478     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8479
8480     my $res = [];
8481     foreach my $id (keys %$data) {
8482         foreach my $item (@{$data->{$id}}) {
8483             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8484             push @$res, $item->{volid} if defined($item->{volid});
8485         }
8486     }
8487
8488     return $res;
8489 }
8490
8491 my $complete_vmid_full = sub {
8492     my ($running) = @_;
8493
8494     my $idlist = vmstatus();
8495
8496     my $res = [];
8497
8498     foreach my $id (keys %$idlist) {
8499         my $d = $idlist->{$id};
8500         if (defined($running)) {
8501             next if $d->{template};
8502             next if $running && $d->{status} ne 'running';
8503             next if !$running && $d->{status} eq 'running';
8504         }
8505         push @$res, $id;
8506
8507     }
8508     return $res;
8509 };
8510
8511 sub complete_vmid {
8512     return &$complete_vmid_full();
8513 }
8514
8515 sub complete_vmid_stopped {
8516     return &$complete_vmid_full(0);
8517 }
8518
8519 sub complete_vmid_running {
8520     return &$complete_vmid_full(1);
8521 }
8522
8523 sub complete_storage {
8524
8525     my $cfg = PVE::Storage::config();
8526     my $ids = $cfg->{ids};
8527
8528     my $res = [];
8529     foreach my $sid (keys %$ids) {
8530         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8531         next if !$ids->{$sid}->{content}->{images};
8532         push @$res, $sid;
8533     }
8534
8535     return $res;
8536 }
8537
8538 sub complete_migration_storage {
8539     my ($cmd, $param, $current_value, $all_args) = @_;
8540
8541     my $targetnode = @$all_args[1];
8542
8543     my $cfg = PVE::Storage::config();
8544     my $ids = $cfg->{ids};
8545
8546     my $res = [];
8547     foreach my $sid (keys %$ids) {
8548         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8549         next if !$ids->{$sid}->{content}->{images};
8550         push @$res, $sid;
8551     }
8552
8553     return $res;
8554 }
8555
8556 sub vm_is_paused {
8557     my ($vmid, $include_suspended) = @_;
8558     my $qmpstatus = eval {
8559         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8560         mon_cmd($vmid, "query-status");
8561     };
8562     warn "$@\n" if $@;
8563     return $qmpstatus && (
8564         $qmpstatus->{status} eq "paused" ||
8565         $qmpstatus->{status} eq "prelaunch" ||
8566         ($include_suspended && $qmpstatus->{status} eq "suspended")
8567     );
8568 }
8569
8570 sub check_volume_storage_type {
8571     my ($storecfg, $vol) = @_;
8572
8573     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8574     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8575     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8576
8577     die "storage '$storeid' does not support content-type '$vtype'\n"
8578         if !$scfg->{content}->{$vtype};
8579
8580     return 1;
8581 }
8582
8583 sub add_nets_bridge_fdb {
8584     my ($conf, $vmid) = @_;
8585
8586     for my $opt (keys %$conf) {
8587         next if $opt !~ m/^net(\d+)$/;
8588         my $iface = "tap${vmid}i$1";
8589         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8590         my $net = parse_net($conf->{$opt}, 1) or next;
8591
8592         my $mac = $net->{macaddr};
8593         if (!$mac) {
8594             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8595                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8596             next;
8597         }
8598
8599         my $bridge = $net->{bridge};
8600         if (!$bridge) {
8601             log_warn("Interface '$iface' not attached to any bridge.");
8602             next;
8603         }
8604         if ($have_sdn) {
8605             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8606         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8607             PVE::Network::add_bridge_fdb($iface, $mac);
8608         }
8609     }
8610 }
8611
8612 sub del_nets_bridge_fdb {
8613     my ($conf, $vmid) = @_;
8614
8615     for my $opt (keys %$conf) {
8616         next if $opt !~ m/^net(\d+)$/;
8617         my $iface = "tap${vmid}i$1";
8618
8619         my $net = parse_net($conf->{$opt}) or next;
8620         my $mac = $net->{macaddr} or next;
8621
8622         my $bridge = $net->{bridge};
8623         if ($have_sdn) {
8624             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8625         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8626             PVE::Network::del_bridge_fdb($iface, $mac);
8627         }
8628     }
8629 }
8630
8631 sub create_ifaces_ipams_ips {
8632     my ($conf, $vmid) = @_;
8633
8634     return if !$have_sdn;
8635
8636     foreach my $opt (keys %$conf) {
8637         if ($opt =~ m/^net(\d+)$/) {
8638             my $value = $conf->{$opt};
8639             my $net = PVE::QemuServer::parse_net($value);
8640             eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8641             warn $@ if $@;
8642         }
8643     }
8644 }
8645
8646 sub delete_ifaces_ipams_ips {
8647     my ($conf, $vmid) = @_;
8648
8649     return if !$have_sdn;
8650
8651     foreach my $opt (keys %$conf) {
8652         if ($opt =~ m/^net(\d+)$/) {
8653             my $net = PVE::QemuServer::parse_net($conf->{$opt});
8654             eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8655             warn $@ if $@;
8656         }
8657     }
8658 }
8659
8660 1;