PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory qw(get_current_memory);
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  62 use PVE::QemuServer::USB;
  63
  64 my $have_sdn;
  65 eval {
  66     require PVE::Network::SDN::Zones;
  67     require PVE::Network::SDN::Vnets;
  68     $have_sdn = 1;
  69 };
  70
  71 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  72 my $OVMF = {
  73     x86_64 => {
  74         '4m-no-smm' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  77         ],
  78         '4m-no-smm-ms' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  81         ],
  82         '4m' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  85         ],
  86         '4m-ms' => [
  87             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  89         ],
  90         # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
  91         # anymore. how can we deperacate this sanely without breaking existing instances, or using
  92         # older backups and snapshot?
  93         default => [
  94             "$EDK2_FW_BASE/OVMF_CODE.fd",
  95             "$EDK2_FW_BASE/OVMF_VARS.fd",
  96         ],
  97     },
  98     aarch64 => {
  99         default => [
 100             "$EDK2_FW_BASE/AAVMF_CODE.fd",
 101             "$EDK2_FW_BASE/AAVMF_VARS.fd",
 102         ],
 103     },
 104 };
 105
 106 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 107
 108 # Note about locking: we use flock on the config file protect against concurent actions.
 109 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 110 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 111 # But you can ignore this kind of lock with the --skiplock flag.
 112
 113 cfs_register_file(
 114     '/qemu-server/',
 115     \&parse_vm_config,
 116     \&write_vm_config
 117 );
 118
 119 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 120     description => "Some command save/restore state from this location.",
 121     type => 'string',
 122     maxLength => 128,
 123     optional => 1,
 124 });
 125
 126 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 127         description => "Specifies the QEMU machine type.",
 128         type => 'string',
 129         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 130         maxLength => 40,
 131         optional => 1,
 132 });
 133
 134 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 135 my $nodename_cache;
 136 sub nodename {
 137     $nodename_cache //= PVE::INotify::nodename();
 138     return $nodename_cache;
 139 }
 140
 141 my $watchdog_fmt = {
 142     model => {
 143         default_key => 1,
 144         type => 'string',
 145         enum => [qw(i6300esb ib700)],
 146         description => "Watchdog type to emulate.",
 147         default => 'i6300esb',
 148         optional => 1,
 149     },
 150     action => {
 151         type => 'string',
 152         enum => [qw(reset shutdown poweroff pause debug none)],
 153         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 154         optional => 1,
 155     },
 156 };
 157 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 158
 159 my $agent_fmt = {
 160     enabled => {
 161         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 162         type => 'boolean',
 163         default => 0,
 164         default_key => 1,
 165     },
 166     fstrim_cloned_disks => {
 167         description => "Run fstrim after moving a disk or migrating the VM.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0,
 171     },
 172     'freeze-fs-on-backup' => {
 173         description => "Freeze/thaw guest filesystems on backup for consistency.",
 174         type => 'boolean',
 175         optional => 1,
 176         default => 1,
 177     },
 178     type => {
 179         description => "Select the agent type",
 180         type => 'string',
 181         default => 'virtio',
 182         optional => 1,
 183         enum => [qw(virtio isa)],
 184     },
 185 };
 186
 187 my $vga_fmt = {
 188     type => {
 189         description => "Select the VGA type.",
 190         type => 'string',
 191         default => 'std',
 192         optional => 1,
 193         default_key => 1,
 194         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 195     },
 196     memory => {
 197         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 198         type => 'integer',
 199         optional => 1,
 200         minimum => 4,
 201         maximum => 512,
 202     },
 203     clipboard => {
 204         description => 'Enable a specific clipboard. If not set, depending on the display type the'
 205             .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
 206         type => 'string',
 207         enum => ['vnc'],
 208         optional => 1,
 209     },
 210 };
 211
 212 my $ivshmem_fmt = {
 213     size => {
 214         type => 'integer',
 215         minimum => 1,
 216         description => "The size of the file in MB.",
 217     },
 218     name => {
 219         type => 'string',
 220         pattern => '[a-zA-Z0-9\-]+',
 221         optional => 1,
 222         format_description => 'string',
 223         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 224     },
 225 };
 226
 227 my $audio_fmt = {
 228     device => {
 229         type => 'string',
 230         enum => [qw(ich9-intel-hda intel-hda AC97)],
 231         description =>  "Configure an audio device."
 232     },
 233     driver =>  {
 234         type => 'string',
 235         enum => ['spice', 'none'],
 236         default => 'spice',
 237         optional => 1,
 238         description => "Driver backend for the audio device."
 239     },
 240 };
 241
 242 my $spice_enhancements_fmt = {
 243     foldersharing => {
 244         type => 'boolean',
 245         optional => 1,
 246         default => '0',
 247         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 248     },
 249     videostreaming =>  {
 250         type => 'string',
 251         enum => ['off', 'all', 'filter'],
 252         default => 'off',
 253         optional => 1,
 254         description => "Enable video streaming. Uses compression for detected video streams."
 255     },
 256 };
 257
 258 my $rng_fmt = {
 259     source => {
 260         type => 'string',
 261         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 262         default_key => 1,
 263         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 264             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 265             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 266             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 267             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 268             ." a hardware RNG from the host.",
 269     },
 270     max_bytes => {
 271         type => 'integer',
 272         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 273             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 274             ." `0` to disable limiting (potentially dangerous!).",
 275         optional => 1,
 276
 277         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 278         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 279         # reading from /dev/urandom
 280         default => 1024,
 281     },
 282     period => {
 283         type => 'integer',
 284         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 285             ." the guest to retrieve another 'max_bytes' of entropy.",
 286         optional => 1,
 287         default => 1000,
 288     },
 289 };
 290
 291 my $meta_info_fmt = {
 292     'ctime' => {
 293         type => 'integer',
 294         description => "The guest creation timestamp as UNIX epoch time",
 295         minimum => 0,
 296         optional => 1,
 297     },
 298     'creation-qemu' => {
 299         type => 'string',
 300         description => "The QEMU (machine) version from the time this VM was created.",
 301         pattern => '\d+(\.\d+)+',
 302         optional => 1,
 303     },
 304 };
 305
 306 my $confdesc = {
 307     onboot => {
 308         optional => 1,
 309         type => 'boolean',
 310         description => "Specifies whether a VM will be started during system bootup.",
 311         default => 0,
 312     },
 313     autostart => {
 314         optional => 1,
 315         type => 'boolean',
 316         description => "Automatic restart after crash (currently ignored).",
 317         default => 0,
 318     },
 319     hotplug => {
 320         optional => 1,
 321         type => 'string', format => 'pve-hotplug-features',
 322         description => "Selectively enable hotplug features. This is a comma separated list of"
 323             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 324             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 325             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 326             ." windows > 7.",
 327         default => 'network,disk,usb',
 328     },
 329     reboot => {
 330         optional => 1,
 331         type => 'boolean',
 332         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 333         default => 1,
 334     },
 335     lock => {
 336         optional => 1,
 337         type => 'string',
 338         description => "Lock/unlock the VM.",
 339         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 340     },
 341     cpulimit => {
 342         optional => 1,
 343         type => 'number',
 344         description => "Limit of CPU usage.",
 345         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 346             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 347         minimum => 0,
 348         maximum => 128,
 349         default => 0,
 350     },
 351     cpuunits => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 355         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 356             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 357             ." weights of all the other running VMs.",
 358         minimum => 1,
 359         maximum => 262144,
 360         default => 'cgroup v1: 1024, cgroup v2: 100',
 361     },
 362     memory => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Memory properties.",
 366         format => $PVE::QemuServer::Memory::memory_fmt
 367     },
 368     balloon => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 372         minimum => 0,
 373     },
 374     shares => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 378             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 379             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 380         minimum => 0,
 381         maximum => 50000,
 382         default => 1000,
 383     },
 384     keyboard => {
 385         optional => 1,
 386         type => 'string',
 387         description => "Keyboard layout for VNC server. This option is generally not required and"
 388         ." is often better handled from within the guest OS.",
 389         enum => PVE::Tools::kvmkeymaplist(),
 390         default => undef,
 391     },
 392     name => {
 393         optional => 1,
 394         type => 'string', format => 'dns-name',
 395         description => "Set a name for the VM. Only used on the configuration web interface.",
 396     },
 397     scsihw => {
 398         optional => 1,
 399         type => 'string',
 400         description => "SCSI controller model",
 401         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 402         default => 'lsi',
 403     },
 404     description => {
 405         optional => 1,
 406         type => 'string',
 407         description => "Description for the VM. Shown in the web-interface VM's summary."
 408             ." This is saved as comment inside the configuration file.",
 409         maxLength => 1024 * 8,
 410     },
 411     ostype => {
 412         optional => 1,
 413         type => 'string',
 414         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 415         description => "Specify guest operating system.",
 416         verbose_description => <<EODESC,
 417 Specify guest operating system. This is used to enable special
 418 optimization/features for specific operating systems:
 419
 420 [horizontal]
 421 other;; unspecified OS
 422 wxp;; Microsoft Windows XP
 423 w2k;; Microsoft Windows 2000
 424 w2k3;; Microsoft Windows 2003
 425 w2k8;; Microsoft Windows 2008
 426 wvista;; Microsoft Windows Vista
 427 win7;; Microsoft Windows 7
 428 win8;; Microsoft Windows 8/2012/2012r2
 429 win10;; Microsoft Windows 10/2016/2019
 430 win11;; Microsoft Windows 11/2022
 431 l24;; Linux 2.4 Kernel
 432 l26;; Linux 2.6 - 6.X Kernel
 433 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 434 EODESC
 435     },
 436     boot => {
 437         optional => 1,
 438         type => 'string', format => 'pve-qm-boot',
 439         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 440             ." key or 'legacy=' is deprecated.",
 441     },
 442     bootdisk => {
 443         optional => 1,
 444         type => 'string', format => 'pve-qm-bootdisk',
 445         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 446         pattern => '(ide|sata|scsi|virtio)\d+',
 447     },
 448     smp => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "The number of CPUs. Please use option -sockets instead.",
 452         minimum => 1,
 453         default => 1,
 454     },
 455     sockets => {
 456         optional => 1,
 457         type => 'integer',
 458         description => "The number of CPU sockets.",
 459         minimum => 1,
 460         default => 1,
 461     },
 462     cores => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "The number of cores per socket.",
 466         minimum => 1,
 467         default => 1,
 468     },
 469     numa => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable NUMA.",
 473         default => 0,
 474     },
 475     hugepages => {
 476         optional => 1,
 477         type => 'string',
 478         description => "Enable/disable hugepages memory.",
 479         enum => [qw(any 2 1024)],
 480     },
 481     keephugepages => {
 482         optional => 1,
 483         type => 'boolean',
 484         default => 0,
 485         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 486             ." after VM shutdown and can be used for subsequent starts.",
 487     },
 488     vcpus => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Number of hotplugged vcpus.",
 492         minimum => 1,
 493         default => 0,
 494     },
 495     acpi => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable ACPI.",
 499         default => 1,
 500     },
 501     agent => {
 502         optional => 1,
 503         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 504         type => 'string',
 505         format => $agent_fmt,
 506     },
 507     kvm => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable KVM hardware virtualization.",
 511         default => 1,
 512     },
 513     tdf => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable time drift fix.",
 517         default => 0,
 518     },
 519     localtime => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 523             ." the `ostype` indicates a Microsoft Windows OS.",
 524     },
 525     freeze => {
 526         optional => 1,
 527         type => 'boolean',
 528         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 529     },
 530     vga => {
 531         optional => 1,
 532         type => 'string', format => $vga_fmt,
 533         description => "Configure the VGA hardware.",
 534         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 535             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 536             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 537             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 538             ." display server. For win* OS you can select how many independent displays you want,"
 539             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 540             ." using a serial device as terminal.",
 541     },
 542     watchdog => {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-watchdog',
 545         description => "Create a virtual hardware watchdog device.",
 546         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 547             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 548             ." else the watchdog will reset the guest (or execute the respective action specified)",
 549     },
 550     startdate => {
 551         optional => 1,
 552         type => 'string',
 553         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 554         description => "Set the initial date of the real time clock. Valid format for date are:"
 555             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 556         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 557         default => 'now',
 558     },
 559     startup =>  get_standard_option('pve-startup-order'),
 560     template => {
 561         optional => 1,
 562         type => 'boolean',
 563         description => "Enable/disable Template.",
 564         default => 0,
 565     },
 566     args => {
 567         optional => 1,
 568         type => 'string',
 569         description => "Arbitrary arguments passed to kvm.",
 570         verbose_description => <<EODESCR,
 571 Arbitrary arguments passed to kvm, for example:
 572
 573 args: -no-reboot -smbios 'type=0,vendor=FOO'
 574
 575 NOTE: this option is for experts only.
 576 EODESCR
 577     },
 578     tablet => {
 579         optional => 1,
 580         type => 'boolean',
 581         default => 1,
 582         description => "Enable/disable the USB tablet device.",
 583         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 584             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 585             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 586             ." may consider disabling this to save some context switches. This is turned off by"
 587             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 588     },
 589     migrate_speed => {
 590         optional => 1,
 591         type => 'integer',
 592         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 593         minimum => 0,
 594         default => 0,
 595     },
 596     migrate_downtime => {
 597         optional => 1,
 598         type => 'number',
 599         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 600         minimum => 0,
 601         default => 0.1,
 602     },
 603     cdrom => {
 604         optional => 1,
 605         type => 'string', format => 'pve-qm-ide',
 606         typetext => '<volume>',
 607         description => "This is an alias for option -ide2",
 608     },
 609     cpu => {
 610         optional => 1,
 611         description => "Emulated CPU type.",
 612         type => 'string',
 613         format => 'pve-vm-cpu-conf',
 614     },
 615     parent => get_standard_option('pve-snapshot-name', {
 616         optional => 1,
 617         description => "Parent snapshot name. This is used internally, and should not be modified.",
 618     }),
 619     snaptime => {
 620         optional => 1,
 621         description => "Timestamp for snapshots.",
 622         type => 'integer',
 623         minimum => 0,
 624     },
 625     vmstate => {
 626         optional => 1,
 627         type => 'string', format => 'pve-volume-id',
 628         description => "Reference to a volume which stores the VM state. This is used internally"
 629             ." for snapshots.",
 630     },
 631     vmstatestorage => get_standard_option('pve-storage-id', {
 632         description => "Default storage for VM state volumes/files.",
 633         optional => 1,
 634     }),
 635     runningmachine => get_standard_option('pve-qemu-machine', {
 636         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 637             ." for snapshots.",
 638     }),
 639     runningcpu => {
 640         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 641             ." internally for snapshots.",
 642         optional => 1,
 643         type => 'string',
 644         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 645         format_description => 'QEMU -cpu parameter'
 646     },
 647     machine => get_standard_option('pve-qemu-machine'),
 648     arch => {
 649         description => "Virtual processor architecture. Defaults to the host.",
 650         optional => 1,
 651         type => 'string',
 652         enum => [qw(x86_64 aarch64)],
 653     },
 654     smbios1 => {
 655         description => "Specify SMBIOS type 1 fields.",
 656         type => 'string', format => 'pve-qm-smbios1',
 657         maxLength => 512,
 658         optional => 1,
 659     },
 660     protection => {
 661         optional => 1,
 662         type => 'boolean',
 663         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 664             ." remove disk operations.",
 665         default => 0,
 666     },
 667     bios => {
 668         optional => 1,
 669         type => 'string',
 670         enum => [ qw(seabios ovmf) ],
 671         description => "Select BIOS implementation.",
 672         default => 'seabios',
 673     },
 674     vmgenid => {
 675         type => 'string',
 676         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 677         format_description => 'UUID',
 678         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 679             ." to disable explicitly.",
 680         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 681             ." value identifier to the guest OS. This allows to notify the guest operating system"
 682             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 683             ." execution or creation from a template). The guest operating system notices the"
 684             ." change, and is then able to react as appropriate by marking its copies of"
 685             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 686             ."Note that auto-creation only works when done through API/CLI create or update methods"
 687             .", but not when manually editing the config file.",
 688         default => "1 (autogenerated)",
 689         optional => 1,
 690     },
 691     hookscript => {
 692         type => 'string',
 693         format => 'pve-volume-id',
 694         optional => 1,
 695         description => "Script that will be executed during various steps in the vms lifetime.",
 696     },
 697     ivshmem => {
 698         type => 'string',
 699         format => $ivshmem_fmt,
 700         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 701             ." the host.",
 702         optional => 1,
 703     },
 704     audio0 => {
 705         type => 'string',
 706         format => $audio_fmt,
 707         description => "Configure a audio device, useful in combination with QXL/Spice.",
 708         optional => 1
 709     },
 710     spice_enhancements => {
 711         type => 'string',
 712         format => $spice_enhancements_fmt,
 713         description => "Configure additional enhancements for SPICE.",
 714         optional => 1
 715     },
 716     tags => {
 717         type => 'string', format => 'pve-tag-list',
 718         description => 'Tags of the VM. This is only meta information.',
 719         optional => 1,
 720     },
 721     rng0 => {
 722         type => 'string',
 723         format => $rng_fmt,
 724         description => "Configure a VirtIO-based Random Number Generator.",
 725         optional => 1,
 726     },
 727     meta => {
 728         type => 'string',
 729         format => $meta_info_fmt,
 730         description => "Some (read-only) meta-information about this guest.",
 731         optional => 1,
 732     },
 733     affinity => {
 734         type => 'string', format => 'pve-cpuset',
 735         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 736         optional => 1,
 737     },
 738 };
 739
 740 my $cicustom_fmt = {
 741     meta => {
 742         type => 'string',
 743         optional => 1,
 744         description => 'Specify a custom file containing all meta data passed to the VM via"
 745             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 746         format => 'pve-volume-id',
 747         format_description => 'volume',
 748     },
 749     network => {
 750         type => 'string',
 751         optional => 1,
 752         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 753         format => 'pve-volume-id',
 754         format_description => 'volume',
 755     },
 756     user => {
 757         type => 'string',
 758         optional => 1,
 759         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 760         format => 'pve-volume-id',
 761         format_description => 'volume',
 762     },
 763     vendor => {
 764         type => 'string',
 765         optional => 1,
 766         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 767         format => 'pve-volume-id',
 768         format_description => 'volume',
 769     },
 770 };
 771 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 772
 773 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 774 my $confdesc_cloudinit = {
 775     citype => {
 776         optional => 1,
 777         type => 'string',
 778         description => 'Specifies the cloud-init configuration format. The default depends on the'
 779             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 780             .' and `configdrive2` for windows.',
 781         enum => ['configdrive2', 'nocloud', 'opennebula'],
 782     },
 783     ciuser => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: User name to change ssh keys and password for instead of the"
 787             ." image's configured default user.",
 788     },
 789     cipassword => {
 790         optional => 1,
 791         type => 'string',
 792         description => 'cloud-init: Password to assign the user. Using this is generally not'
 793             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 794             .' support hashed passwords.',
 795     },
 796     ciupgrade => {
 797         optional => 1,
 798         type => 'boolean',
 799         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 800         default => 1,
 801     },
 802     cicustom => {
 803         optional => 1,
 804         type => 'string',
 805         description => 'cloud-init: Specify custom files to replace the automatically generated'
 806             .' ones at start.',
 807         format => 'pve-qm-cicustom',
 808     },
 809     searchdomain => {
 810         optional => 1,
 811         type => 'string',
 812         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 813             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 814             .' are set.',
 815     },
 816     nameserver => {
 817         optional => 1,
 818         type => 'string', format => 'address-list',
 819         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 820             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 821             .' are set.',
 822     },
 823     sshkeys => {
 824         optional => 1,
 825         type => 'string',
 826         format => 'urlencoded',
 827         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 828     },
 829 };
 830
 831 # what about other qemu settings ?
 832 #cpu => 'string',
 833 #machine => 'string',
 834 #fda => 'file',
 835 #fdb => 'file',
 836 #mtdblock => 'file',
 837 #sd => 'file',
 838 #pflash => 'file',
 839 #snapshot => 'bool',
 840 #bootp => 'file',
 841 ##tftp => 'dir',
 842 ##smb => 'dir',
 843 #kernel => 'file',
 844 #append => 'string',
 845 #initrd => 'file',
 846 ##soundhw => 'string',
 847
 848 while (my ($k, $v) = each %$confdesc) {
 849     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 850 }
 851
 852 my $MAX_NETS = 32;
 853 my $MAX_SERIAL_PORTS = 4;
 854 my $MAX_PARALLEL_PORTS = 3;
 855
 856 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 857     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 858 }
 859
 860 my $nic_model_list = [
 861     'e1000',
 862     'e1000-82540em',
 863     'e1000-82544gc',
 864     'e1000-82545em',
 865     'e1000e',
 866     'i82551',
 867     'i82557b',
 868     'i82559er',
 869     'ne2k_isa',
 870     'ne2k_pci',
 871     'pcnet',
 872     'rtl8139',
 873     'virtio',
 874     'vmxnet3',
 875 ];
 876 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 877
 878 my $net_fmt_bridge_descr = <<__EOD__;
 879 Bridge to attach the network device to. The Proxmox VE standard bridge
 880 is called 'vmbr0'.
 881
 882 If you do not specify a bridge, we create a kvm user (NATed) network
 883 device, which provides DHCP and DNS services. The following addresses
 884 are used:
 885
 886  10.0.2.2   Gateway
 887  10.0.2.3   DNS Server
 888  10.0.2.4   SMB Server
 889
 890 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 891 __EOD__
 892
 893 my $net_fmt = {
 894     macaddr  => get_standard_option('mac-addr', {
 895         description => "MAC address. That address must be unique withing your network. This is"
 896             ." automatically generated if not specified.",
 897     }),
 898     model => {
 899         type => 'string',
 900         description => "Network Card Model. The 'virtio' model provides the best performance with"
 901             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 902             ." best to use 'e1000'.",
 903         enum => $nic_model_list,
 904         default_key => 1,
 905     },
 906     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 907     bridge => get_standard_option('pve-bridge-id', {
 908         description => $net_fmt_bridge_descr,
 909         optional => 1,
 910     }),
 911     queues => {
 912         type => 'integer',
 913         minimum => 0, maximum => 64,
 914         description => 'Number of packet queues to be used on the device.',
 915         optional => 1,
 916     },
 917     rate => {
 918         type => 'number',
 919         minimum => 0,
 920         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 921         optional => 1,
 922     },
 923     tag => {
 924         type => 'integer',
 925         minimum => 1, maximum => 4094,
 926         description => 'VLAN tag to apply to packets on this interface.',
 927         optional => 1,
 928     },
 929     trunks => {
 930         type => 'string',
 931         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 932         description => 'VLAN trunks to pass through this interface.',
 933         format_description => 'vlanid[;vlanid...]',
 934         optional => 1,
 935     },
 936     firewall => {
 937         type => 'boolean',
 938         description => 'Whether this interface should be protected by the firewall.',
 939         optional => 1,
 940     },
 941     link_down => {
 942         type => 'boolean',
 943         description => 'Whether this interface should be disconnected (like pulling the plug).',
 944         optional => 1,
 945     },
 946     mtu => {
 947         type => 'integer',
 948         minimum => 1, maximum => 65520,
 949         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 950         optional => 1,
 951     },
 952 };
 953
 954 my $netdesc = {
 955     optional => 1,
 956     type => 'string', format => $net_fmt,
 957     description => "Specify network devices.",
 958 };
 959
 960 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 961
 962 my $ipconfig_fmt = {
 963     ip => {
 964         type => 'string',
 965         format => 'pve-ipv4-config',
 966         format_description => 'IPv4Format/CIDR',
 967         description => 'IPv4 address in CIDR format.',
 968         optional => 1,
 969         default => 'dhcp',
 970     },
 971     gw => {
 972         type => 'string',
 973         format => 'ipv4',
 974         format_description => 'GatewayIPv4',
 975         description => 'Default gateway for IPv4 traffic.',
 976         optional => 1,
 977         requires => 'ip',
 978     },
 979     ip6 => {
 980         type => 'string',
 981         format => 'pve-ipv6-config',
 982         format_description => 'IPv6Format/CIDR',
 983         description => 'IPv6 address in CIDR format.',
 984         optional => 1,
 985         default => 'dhcp',
 986     },
 987     gw6 => {
 988         type => 'string',
 989         format => 'ipv6',
 990         format_description => 'GatewayIPv6',
 991         description => 'Default gateway for IPv6 traffic.',
 992         optional => 1,
 993         requires => 'ip6',
 994     },
 995 };
 996 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 997 my $ipconfigdesc = {
 998     optional => 1,
 999     type => 'string', format => 'pve-qm-ipconfig',
1000     description => <<'EODESCR',
1001 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1002
1003 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1004
1005 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1006 gateway should be provided.
1007 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1008 cloud-init 19.4 or newer.
1009
1010 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1011 dhcp on IPv4.
1012 EODESCR
1013 };
1014 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1015
1016 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1017     $confdesc->{"net$i"} = $netdesc;
1018     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1019 }
1020
1021 foreach my $key (keys %$confdesc_cloudinit) {
1022     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1023 }
1024
1025 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1026 sub pve_verify_cpuset {
1027     my ($set_text, $noerr) = @_;
1028
1029     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1030
1031     if ($@) {
1032         return if $noerr;
1033         die "unable to parse cpuset option\n";
1034     }
1035
1036     return PVE::CpuSet->new($members)->short_string();
1037 }
1038
1039 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1040 sub verify_volume_id_or_qm_path {
1041     my ($volid, $noerr) = @_;
1042
1043     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1044
1045     return verify_volume_id_or_absolute_path($volid, $noerr);
1046 }
1047
1048 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1049 sub verify_volume_id_or_absolute_path {
1050     my ($volid, $noerr) = @_;
1051
1052     return $volid if $volid =~ m|^/|;
1053
1054     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1055     if ($@) {
1056         return if $noerr;
1057         die $@;
1058     }
1059     return $volid;
1060 }
1061
1062 my $serialdesc = {
1063         optional => 1,
1064         type => 'string',
1065         pattern => '(/dev/.+|socket)',
1066         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1067         verbose_description =>  <<EODESCR,
1068 Create a serial device inside the VM (n is 0 to 3), and pass through a
1069 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1070 host side (use 'qm terminal' to open a terminal connection).
1071
1072 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1073 use with special care.
1074
1075 CAUTION: Experimental! User reported problems with this option.
1076 EODESCR
1077 };
1078
1079 my $paralleldesc= {
1080         optional => 1,
1081         type => 'string',
1082         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1083         description =>  "Map host parallel devices (n is 0 to 2).",
1084         verbose_description =>  <<EODESCR,
1085 Map host parallel devices (n is 0 to 2).
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1095     $confdesc->{"parallel$i"} = $paralleldesc;
1096 }
1097
1098 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1099     $confdesc->{"serial$i"} = $serialdesc;
1100 }
1101
1102 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1103     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1104 }
1105
1106 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1107     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1108 }
1109
1110 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1111     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1112 }
1113
1114 my $boot_fmt = {
1115     legacy => {
1116         optional => 1,
1117         default_key => 1,
1118         type => 'string',
1119         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1120                      . " Deprecated, use 'order=' instead.",
1121         pattern => '[acdn]{1,4}',
1122         format_description => "[acdn]{1,4}",
1123
1124         # note: this is also the fallback if boot: is not given at all
1125         default => 'cdn',
1126     },
1127     order => {
1128         optional => 1,
1129         type => 'string',
1130         format => 'pve-qm-bootdev-list',
1131         format_description => "device[;device...]",
1132         description => <<EODESC,
1133 The guest will attempt to boot from devices in the order they appear here.
1134
1135 Disks, optical drives and passed-through storage USB devices will be directly
1136 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1137 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1138
1139 Note that only devices in this list will be marked as bootable and thus loaded
1140 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1141 (e.g. software-raid), you need to specify all of them here.
1142
1143 Overrides the deprecated 'legacy=[acdn]*' value when given.
1144 EODESC
1145     },
1146 };
1147 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1148
1149 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1150 sub verify_bootdev {
1151     my ($dev, $noerr) = @_;
1152
1153     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1154     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1155
1156     my $check = sub {
1157         my ($base) = @_;
1158         return 0 if $dev !~ m/^$base\d+$/;
1159         return 0 if !$confdesc->{$dev};
1160         return 1;
1161     };
1162
1163     return $dev if $check->("net");
1164     return $dev if $check->("usb");
1165     return $dev if $check->("hostpci");
1166
1167     return if $noerr;
1168     die "invalid boot device '$dev'\n";
1169 }
1170
1171 sub print_bootorder {
1172     my ($devs) = @_;
1173     return "" if !@$devs;
1174     my $data = { order => join(';', @$devs) };
1175     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1176 }
1177
1178 my $kvm_api_version = 0;
1179
1180 sub kvm_version {
1181     return $kvm_api_version if $kvm_api_version;
1182
1183     open my $fh, '<', '/dev/kvm' or return;
1184
1185     # 0xae00 => KVM_GET_API_VERSION
1186     $kvm_api_version = ioctl($fh, 0xae00, 0);
1187     close($fh);
1188
1189     return $kvm_api_version;
1190 }
1191
1192 my $kvm_user_version = {};
1193 my $kvm_mtime = {};
1194
1195 sub kvm_user_version {
1196     my ($binary) = @_;
1197
1198     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1199     my $st = stat($binary);
1200
1201     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1202     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1203         $cachedmtime == $st->mtime;
1204
1205     $kvm_user_version->{$binary} = 'unknown';
1206     $kvm_mtime->{$binary} = $st->mtime;
1207
1208     my $code = sub {
1209         my $line = shift;
1210         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1211             $kvm_user_version->{$binary} = $2;
1212         }
1213     };
1214
1215     eval { run_command([$binary, '--version'], outfunc => $code); };
1216     warn $@ if $@;
1217
1218     return $kvm_user_version->{$binary};
1219
1220 }
1221 my sub extract_version {
1222     my ($machine_type, $version) = @_;
1223     $version = kvm_user_version() if !defined($version);
1224     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1225 }
1226
1227 sub kernel_has_vhost_net {
1228     return -c '/dev/vhost-net';
1229 }
1230
1231 sub option_exists {
1232     my $key = shift;
1233     return defined($confdesc->{$key});
1234 }
1235
1236 my $cdrom_path;
1237 sub get_cdrom_path {
1238
1239     return $cdrom_path if defined($cdrom_path);
1240
1241     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1242
1243     if (!defined($cdrom_path)) {
1244         log_warn("no physical CD-ROM available, ignoring");
1245         $cdrom_path = '';
1246     }
1247
1248     return $cdrom_path;
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269      if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1314             if !$vtype;
1315         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1316         verify_media_type($opt, $vtype, $drive->{media});
1317         $drive->{file} = $volid;
1318     }
1319
1320     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1321 }
1322
1323 sub parse_hotplug_features {
1324     my ($data) = @_;
1325
1326     my $res = {};
1327
1328     return $res if $data eq '0';
1329
1330     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1331
1332     foreach my $feature (PVE::Tools::split_list($data)) {
1333         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1334             $res->{$1} = 1;
1335         } else {
1336             die "invalid hotplug feature '$feature'\n";
1337         }
1338     }
1339     return $res;
1340 }
1341
1342 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1343 sub pve_verify_hotplug_features {
1344     my ($value, $noerr) = @_;
1345
1346     return $value if parse_hotplug_features($value);
1347
1348     return if $noerr;
1349
1350     die "unable to parse hotplug option\n";
1351 }
1352
1353 sub assert_clipboard_config {
1354     my ($vga) = @_;
1355
1356     my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1357
1358     if (
1359         $vga->{'clipboard'}
1360         && $vga->{'clipboard'} eq 'vnc'
1361         && $vga->{type}
1362         && $vga->{type} !~ $clipboard_regex
1363     ) {
1364         die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1365     }
1366 }
1367
1368 sub print_tabletdevice_full {
1369     my ($conf, $arch) = @_;
1370
1371     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1372
1373     # we use uhci for old VMs because tablet driver was buggy in older qemu
1374     my $usbbus;
1375     if ($q35 || $arch eq 'aarch64') {
1376         $usbbus = 'ehci';
1377     } else {
1378         $usbbus = 'uhci';
1379     }
1380
1381     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1382 }
1383
1384 sub print_keyboarddevice_full {
1385     my ($conf, $arch) = @_;
1386
1387     return if $arch ne 'aarch64';
1388
1389     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1390 }
1391
1392 my sub get_drive_id {
1393     my ($drive) = @_;
1394     return "$drive->{interface}$drive->{index}";
1395 }
1396
1397 sub print_drivedevice_full {
1398     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1399
1400     my $device = '';
1401     my $maxdev = 0;
1402
1403     my $drive_id = get_drive_id($drive);
1404     if ($drive->{interface} eq 'virtio') {
1405         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1406         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1407         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1408     } elsif ($drive->{interface} eq 'scsi') {
1409
1410         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1411         my $unit = $drive->{index} % $maxdev;
1412
1413         my $machine_version = extract_version($machine_type, kvm_user_version());
1414         my $devicetype  = PVE::QemuServer::Drive::get_scsi_devicetype(
1415             $drive, $storecfg, $machine_version);
1416
1417         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1418             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1419         } else {
1420             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1421                 .",lun=$drive->{index}";
1422         }
1423         $device .= ",drive=drive-$drive_id,id=$drive_id";
1424
1425         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1426             $device .= ",rotation_rate=1";
1427         }
1428         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1429
1430         # only scsi-hd and scsi-cd support passing vendor and product information
1431         if ($devicetype eq 'hd' || $devicetype eq 'cd') {
1432             if (my $vendor = $drive->{vendor}) {
1433                 $device .= ",vendor=$vendor";
1434             }
1435             if (my $product = $drive->{product}) {
1436                 $device .= ",product=$product";
1437             }
1438         }
1439
1440     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1441         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1442         my $controller = int($drive->{index} / $maxdev);
1443         my $unit = $drive->{index} % $maxdev;
1444
1445         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1446         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1447         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1448         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1449         # were before. Move odd ones up by 2 where they don't clash.
1450         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1451             $controller += 2 * ($unit % 2);
1452             $unit = 0;
1453         }
1454
1455         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1456
1457         $device = "ide-$devicetype";
1458         if ($drive->{interface} eq 'ide') {
1459             $device .= ",bus=ide.$controller,unit=$unit";
1460         } else {
1461             $device .= ",bus=ahci$controller.$unit";
1462         }
1463         $device .= ",drive=drive-$drive_id,id=$drive_id";
1464
1465         if ($devicetype eq 'hd') {
1466             if (my $model = $drive->{model}) {
1467                 $model = URI::Escape::uri_unescape($model);
1468                 $device .= ",model=$model";
1469             }
1470             if ($drive->{ssd}) {
1471                 $device .= ",rotation_rate=1";
1472             }
1473         }
1474         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1475     } elsif ($drive->{interface} eq 'usb') {
1476         die "implement me";
1477         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1478     } else {
1479         die "unsupported interface type";
1480     }
1481
1482     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1483
1484     if (my $serial = $drive->{serial}) {
1485         $serial = URI::Escape::uri_unescape($serial);
1486         $device .= ",serial=$serial";
1487     }
1488
1489
1490     return $device;
1491 }
1492
1493 sub get_initiator_name {
1494     my $initiator;
1495
1496     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1497     while (defined(my $line = <$fh>)) {
1498         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1499         $initiator = $1;
1500         last;
1501     }
1502     $fh->close();
1503
1504     return $initiator;
1505 }
1506
1507 my sub storage_allows_io_uring_default {
1508     my ($scfg, $cache_direct) = @_;
1509
1510     # io_uring with cache mode writeback or writethrough on krbd will hang...
1511     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1512
1513     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1514     # sometimes, just plain disable...
1515     return if $scfg && $scfg->{type} eq 'lvm';
1516
1517     # io_uring causes problems when used with CIFS since kernel 5.15
1518     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1519     return if $scfg && $scfg->{type} eq 'cifs';
1520
1521     return 1;
1522 }
1523
1524 my sub drive_uses_cache_direct {
1525     my ($drive, $scfg) = @_;
1526
1527     my $cache_direct = 0;
1528
1529     if (my $cache = $drive->{cache}) {
1530         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1531     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1532         $cache_direct = 1;
1533     }
1534
1535     return $cache_direct;
1536 }
1537
1538 sub print_drive_commandline_full {
1539     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1540
1541     my $path;
1542     my $volid = $drive->{file};
1543     my $format = $drive->{format};
1544     my $drive_id = get_drive_id($drive);
1545
1546     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1547     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1548
1549     if (drive_is_cdrom($drive)) {
1550         $path = get_iso_path($storecfg, $vmid, $volid);
1551         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1552     } else {
1553         if ($storeid) {
1554             $path = PVE::Storage::path($storecfg, $volid);
1555             $format //= qemu_img_format($scfg, $volname);
1556         } else {
1557             $path = $volid;
1558             $format //= "raw";
1559         }
1560    }
1561
1562    my $is_rbd = $path =~ m/^rbd:/;
1563
1564     my $opts = '';
1565     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1566     foreach my $o (@qemu_drive_options) {
1567         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1568     }
1569
1570     # snapshot only accepts on|off
1571     if (defined($drive->{snapshot})) {
1572         my $v = $drive->{snapshot} ? 'on' : 'off';
1573         $opts .= ",snapshot=$v";
1574     }
1575
1576     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1577         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1578     }
1579
1580     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1581         my ($dir, $qmpname) = @$type;
1582         if (my $v = $drive->{"mbps$dir"}) {
1583             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1584         }
1585         if (my $v = $drive->{"mbps${dir}_max"}) {
1586             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1587         }
1588         if (my $v = $drive->{"bps${dir}_max_length"}) {
1589             $opts .= ",throttling.bps$qmpname-max-length=$v";
1590         }
1591         if (my $v = $drive->{"iops${dir}"}) {
1592             $opts .= ",throttling.iops$qmpname=$v";
1593         }
1594         if (my $v = $drive->{"iops${dir}_max"}) {
1595             $opts .= ",throttling.iops$qmpname-max=$v";
1596         }
1597         if (my $v = $drive->{"iops${dir}_max_length"}) {
1598             $opts .= ",throttling.iops$qmpname-max-length=$v";
1599         }
1600     }
1601
1602     if ($pbs_name) {
1603         $format = "rbd" if $is_rbd;
1604         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1605             if !$format;
1606         $opts .= ",format=alloc-track,file.driver=$format";
1607     } elsif ($format) {
1608         $opts .= ",format=$format";
1609     }
1610
1611     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1612
1613     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1614
1615     if (!$drive->{aio}) {
1616         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1617             # io_uring supports all cache modes
1618             $opts .= ",aio=io_uring";
1619         } else {
1620             # aio native works only with O_DIRECT
1621             if($cache_direct) {
1622                 $opts .= ",aio=native";
1623             } else {
1624                 $opts .= ",aio=threads";
1625             }
1626         }
1627     }
1628
1629     if (!drive_is_cdrom($drive)) {
1630         my $detectzeroes;
1631         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1632             $detectzeroes = 'off';
1633         } elsif ($drive->{discard}) {
1634             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1635         } else {
1636             # This used to be our default with discard not being specified:
1637             $detectzeroes = 'on';
1638         }
1639
1640         # note: 'detect-zeroes' works per blockdev and we want it to persist
1641         # after the alloc-track is removed, so put it on 'file' directly
1642         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1643         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1644     }
1645
1646     if ($pbs_name) {
1647         $opts .= ",backing=$pbs_name";
1648         $opts .= ",auto-remove=on";
1649     }
1650
1651     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1652     my $file_param = "file";
1653     if ($pbs_name) {
1654         # non-rbd drivers require the underlying file to be a seperate block
1655         # node, so add a second .file indirection
1656         $file_param .= ".file" if !$is_rbd;
1657         $file_param .= ".filename";
1658     }
1659     my $pathinfo = $path ? "$file_param=$path," : '';
1660
1661     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1662 }
1663
1664 sub print_pbs_blockdev {
1665     my ($pbs_conf, $pbs_name) = @_;
1666     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1667     $blockdev .= ",repository=$pbs_conf->{repository}";
1668     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1669     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1670     $blockdev .= ",archive=$pbs_conf->{archive}";
1671     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1672     return $blockdev;
1673 }
1674
1675 sub print_netdevice_full {
1676     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1677
1678     my $device = $net->{model};
1679     if ($net->{model} eq 'virtio') {
1680          $device = 'virtio-net-pci';
1681      };
1682
1683     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1684     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1685     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1686         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1687         # and out of each queue plus one config interrupt and control vector queue
1688         my $vectors = $net->{queues} * 2 + 2;
1689         $tmpstr .= ",vectors=$vectors,mq=on";
1690         if (min_version($machine_version, 7, 1)) {
1691             $tmpstr .= ",packed=on";
1692         }
1693     }
1694
1695     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1696         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1697     }
1698
1699     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1700
1701     if (my $mtu = $net->{mtu}) {
1702         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1703             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1704             if ($mtu == 1) {
1705                  $mtu = $bridge_mtu;
1706             } elsif ($mtu < 576) {
1707                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1708             } elsif ($mtu > $bridge_mtu) {
1709                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1710             }
1711             $tmpstr .= ",host_mtu=$mtu";
1712         } else {
1713             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1714         }
1715     }
1716
1717     if ($use_old_bios_files) {
1718         my $romfile;
1719         if ($device eq 'virtio-net-pci') {
1720             $romfile = 'pxe-virtio.rom';
1721         } elsif ($device eq 'e1000') {
1722             $romfile = 'pxe-e1000.rom';
1723         } elsif ($device eq 'e1000e') {
1724             $romfile = 'pxe-e1000e.rom';
1725         } elsif ($device eq 'ne2k') {
1726             $romfile = 'pxe-ne2k_pci.rom';
1727         } elsif ($device eq 'pcnet') {
1728             $romfile = 'pxe-pcnet.rom';
1729         } elsif ($device eq 'rtl8139') {
1730             $romfile = 'pxe-rtl8139.rom';
1731         }
1732         $tmpstr .= ",romfile=$romfile" if $romfile;
1733     }
1734
1735     return $tmpstr;
1736 }
1737
1738 sub print_netdev_full {
1739     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1740
1741     my $i = '';
1742     if ($netid =~ m/^net(\d+)$/) {
1743         $i = int($1);
1744     }
1745
1746     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1747
1748     my $ifname = "tap${vmid}i$i";
1749
1750     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1751     die "interface name '$ifname' is too long (max 15 character)\n"
1752         if length($ifname) >= 16;
1753
1754     my $vhostparam = '';
1755     if (is_native_arch($arch)) {
1756         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1757     }
1758
1759     my $vmname = $conf->{name} || "vm$vmid";
1760
1761     my $netdev = "";
1762     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1763
1764     if ($net->{bridge}) {
1765         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1766             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1767     } else {
1768         $netdev = "type=user,id=$netid,hostname=$vmname";
1769     }
1770
1771     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1772
1773     return $netdev;
1774 }
1775
1776 my $vga_map = {
1777     'cirrus' => 'cirrus-vga',
1778     'std' => 'VGA',
1779     'vmware' => 'vmware-svga',
1780     'virtio' => 'virtio-vga',
1781     'virtio-gl' => 'virtio-vga-gl',
1782 };
1783
1784 sub print_vga_device {
1785     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1786
1787     my $type = $vga_map->{$vga->{type}};
1788     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1789         $type = 'virtio-gpu';
1790     }
1791     my $vgamem_mb = $vga->{memory};
1792
1793     my $max_outputs = '';
1794     if ($qxlnum) {
1795         $type = $id ? 'qxl' : 'qxl-vga';
1796
1797         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1798             # set max outputs so linux can have up to 4 qxl displays with one device
1799             if (min_version($machine_version, 4, 1)) {
1800                 $max_outputs = ",max_outputs=4";
1801             }
1802         }
1803     }
1804
1805     die "no devicetype for $vga->{type}\n" if !$type;
1806
1807     my $memory = "";
1808     if ($vgamem_mb) {
1809         if ($vga->{type} =~ /^virtio/) {
1810             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1811             $memory = ",max_hostmem=$bytes";
1812         } elsif ($qxlnum) {
1813             # from https://www.spice-space.org/multiple-monitors.html
1814             $memory = ",vgamem_mb=$vga->{memory}";
1815             my $ram = $vgamem_mb * 4;
1816             my $vram = $vgamem_mb * 2;
1817             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1818         } else {
1819             $memory = ",vgamem_mb=$vga->{memory}";
1820         }
1821     } elsif ($qxlnum && $id) {
1822         $memory = ",ram_size=67108864,vram_size=33554432";
1823     }
1824
1825     my $edidoff = "";
1826     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1827         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1828     }
1829
1830     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1831     my $vgaid = "vga" . ($id // '');
1832     my $pciaddr;
1833     if ($q35 && $vgaid eq 'vga') {
1834         # the first display uses pcie.0 bus on q35 machines
1835         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1836     } else {
1837         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1838     }
1839
1840     if ($vga->{type} eq 'virtio-gl') {
1841         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1842         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1843             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1844
1845         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1846             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1847     }
1848
1849     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1850 }
1851
1852 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1853 sub parse_net {
1854     my ($data, $disable_mac_autogen) = @_;
1855
1856     my $res = eval { parse_property_string($net_fmt, $data) };
1857     if ($@) {
1858         warn $@;
1859         return;
1860     }
1861     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1862         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1863         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1864     }
1865     return $res;
1866 }
1867
1868 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1869 sub parse_ipconfig {
1870     my ($data) = @_;
1871
1872     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1873     if ($@) {
1874         warn $@;
1875         return;
1876     }
1877
1878     if ($res->{gw} && !$res->{ip}) {
1879         warn 'gateway specified without specifying an IP address';
1880         return;
1881     }
1882     if ($res->{gw6} && !$res->{ip6}) {
1883         warn 'IPv6 gateway specified without specifying an IPv6 address';
1884         return;
1885     }
1886     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1887         warn 'gateway specified together with DHCP';
1888         return;
1889     }
1890     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1891         # gw6 + auto/dhcp
1892         warn "IPv6 gateway specified together with $res->{ip6} address";
1893         return;
1894     }
1895
1896     if (!$res->{ip} && !$res->{ip6}) {
1897         return { ip => 'dhcp', ip6 => 'dhcp' };
1898     }
1899
1900     return $res;
1901 }
1902
1903 sub print_net {
1904     my $net = shift;
1905
1906     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1907 }
1908
1909 sub add_random_macs {
1910     my ($settings) = @_;
1911
1912     foreach my $opt (keys %$settings) {
1913         next if $opt !~ m/^net(\d+)$/;
1914         my $net = parse_net($settings->{$opt});
1915         next if !$net;
1916         $settings->{$opt} = print_net($net);
1917     }
1918 }
1919
1920 sub vm_is_volid_owner {
1921     my ($storecfg, $vmid, $volid) = @_;
1922
1923     if ($volid !~  m|^/|) {
1924         my ($path, $owner);
1925         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1926         if ($owner && ($owner == $vmid)) {
1927             return 1;
1928         }
1929     }
1930
1931     return;
1932 }
1933
1934 sub vmconfig_register_unused_drive {
1935     my ($storecfg, $vmid, $conf, $drive) = @_;
1936
1937     if (drive_is_cloudinit($drive)) {
1938         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1939         warn $@ if $@;
1940         delete $conf->{cloudinit};
1941     } elsif (!drive_is_cdrom($drive)) {
1942         my $volid = $drive->{file};
1943         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1944             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1945         }
1946     }
1947 }
1948
1949 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1950 my $smbios1_fmt = {
1951     uuid => {
1952         type => 'string',
1953         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1954         format_description => 'UUID',
1955         description => "Set SMBIOS1 UUID.",
1956         optional => 1,
1957     },
1958     version => {
1959         type => 'string',
1960         pattern => '[A-Za-z0-9+\/]+={0,2}',
1961         format_description => 'Base64 encoded string',
1962         description => "Set SMBIOS1 version.",
1963         optional => 1,
1964     },
1965     serial => {
1966         type => 'string',
1967         pattern => '[A-Za-z0-9+\/]+={0,2}',
1968         format_description => 'Base64 encoded string',
1969         description => "Set SMBIOS1 serial number.",
1970         optional => 1,
1971     },
1972     manufacturer => {
1973         type => 'string',
1974         pattern => '[A-Za-z0-9+\/]+={0,2}',
1975         format_description => 'Base64 encoded string',
1976         description => "Set SMBIOS1 manufacturer.",
1977         optional => 1,
1978     },
1979     product => {
1980         type => 'string',
1981         pattern => '[A-Za-z0-9+\/]+={0,2}',
1982         format_description => 'Base64 encoded string',
1983         description => "Set SMBIOS1 product ID.",
1984         optional => 1,
1985     },
1986     sku => {
1987         type => 'string',
1988         pattern => '[A-Za-z0-9+\/]+={0,2}',
1989         format_description => 'Base64 encoded string',
1990         description => "Set SMBIOS1 SKU string.",
1991         optional => 1,
1992     },
1993     family => {
1994         type => 'string',
1995         pattern => '[A-Za-z0-9+\/]+={0,2}',
1996         format_description => 'Base64 encoded string',
1997         description => "Set SMBIOS1 family string.",
1998         optional => 1,
1999     },
2000     base64 => {
2001         type => 'boolean',
2002         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2003         optional => 1,
2004     },
2005 };
2006
2007 sub parse_smbios1 {
2008     my ($data) = @_;
2009
2010     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2011     warn $@ if $@;
2012     return $res;
2013 }
2014
2015 sub print_smbios1 {
2016     my ($smbios1) = @_;
2017     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2018 }
2019
2020 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2021
2022 sub parse_watchdog {
2023     my ($value) = @_;
2024
2025     return if !$value;
2026
2027     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2028     warn $@ if $@;
2029     return $res;
2030 }
2031
2032 sub parse_guest_agent {
2033     my ($conf) = @_;
2034
2035     return {} if !defined($conf->{agent});
2036
2037     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2038     warn $@ if $@;
2039
2040     # if the agent is disabled ignore the other potentially set properties
2041     return {} if !$res->{enabled};
2042     return $res;
2043 }
2044
2045 sub get_qga_key {
2046     my ($conf, $key) = @_;
2047     return undef if !defined($conf->{agent});
2048
2049     my $agent = parse_guest_agent($conf);
2050     return $agent->{$key};
2051 }
2052
2053 sub parse_vga {
2054     my ($value) = @_;
2055
2056     return {} if !$value;
2057     my $res = eval { parse_property_string($vga_fmt, $value) };
2058     warn $@ if $@;
2059     return $res;
2060 }
2061
2062 sub parse_rng {
2063     my ($value) = @_;
2064
2065     return if !$value;
2066
2067     my $res = eval { parse_property_string($rng_fmt, $value) };
2068     warn $@ if $@;
2069     return $res;
2070 }
2071
2072 sub parse_meta_info {
2073     my ($value) = @_;
2074
2075     return if !$value;
2076
2077     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2078     warn $@ if $@;
2079     return $res;
2080 }
2081
2082 sub new_meta_info_string {
2083     my () = @_; # for now do not allow to override any value
2084
2085     return PVE::JSONSchema::print_property_string(
2086         {
2087             'creation-qemu' => kvm_user_version(),
2088             ctime => "". int(time()),
2089         },
2090         $meta_info_fmt
2091     );
2092 }
2093
2094 sub qemu_created_version_fixups {
2095     my ($conf, $forcemachine, $kvmver) = @_;
2096
2097     my $meta = parse_meta_info($conf->{meta}) // {};
2098     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2099
2100     # check if we need to apply some handling for VMs that always use the latest machine version but
2101     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2102     # would be required (we do not want to pin machine version for non-windows OS type)
2103     if (
2104         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2105         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2106         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2107         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2108     ) {
2109         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2110         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2111             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2112             # and thus with the predictable interface naming of systemd
2113             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2114         }
2115     }
2116     return;
2117 }
2118
2119 # add JSON properties for create and set function
2120 sub json_config_properties {
2121     my ($prop, $with_disk_alloc) = @_;
2122
2123     my $skip_json_config_opts = {
2124         parent => 1,
2125         snaptime => 1,
2126         vmstate => 1,
2127         runningmachine => 1,
2128         runningcpu => 1,
2129         meta => 1,
2130     };
2131
2132     foreach my $opt (keys %$confdesc) {
2133         next if $skip_json_config_opts->{$opt};
2134
2135         if ($with_disk_alloc && is_valid_drivename($opt)) {
2136             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2137         } else {
2138             $prop->{$opt} = $confdesc->{$opt};
2139         }
2140     }
2141
2142     return $prop;
2143 }
2144
2145 # Properties that we can read from an OVF file
2146 sub json_ovf_properties {
2147     my $prop = {};
2148
2149     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2150         $prop->{$device} = {
2151             type => 'string',
2152             format => 'pve-volume-id-or-absolute-path',
2153             description => "Disk image that gets imported to $device",
2154             optional => 1,
2155         };
2156     }
2157
2158     $prop->{cores} = {
2159         type => 'integer',
2160         description => "The number of CPU cores.",
2161         optional => 1,
2162     };
2163     $prop->{memory} = {
2164         type => 'integer',
2165         description => "Amount of RAM for the VM in MB.",
2166         optional => 1,
2167     };
2168     $prop->{name} = {
2169         type => 'string',
2170         description => "Name of the VM.",
2171         optional => 1,
2172     };
2173
2174     return $prop;
2175 }
2176
2177 # return copy of $confdesc_cloudinit to generate documentation
2178 sub cloudinit_config_properties {
2179
2180     return dclone($confdesc_cloudinit);
2181 }
2182
2183 sub cloudinit_pending_properties {
2184     my $p = {
2185         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2186         name => 1,
2187     };
2188     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2189     return $p;
2190 }
2191
2192 sub check_type {
2193     my ($key, $value) = @_;
2194
2195     die "unknown setting '$key'\n" if !$confdesc->{$key};
2196
2197     my $type = $confdesc->{$key}->{type};
2198
2199     if (!defined($value)) {
2200         die "got undefined value\n";
2201     }
2202
2203     if ($value =~ m/[\n\r]/) {
2204         die "property contains a line feed\n";
2205     }
2206
2207     if ($type eq 'boolean') {
2208         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2209         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2210         die "type check ('boolean') failed - got '$value'\n";
2211     } elsif ($type eq 'integer') {
2212         return int($1) if $value =~ m/^(\d+)$/;
2213         die "type check ('integer') failed - got '$value'\n";
2214     } elsif ($type eq 'number') {
2215         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2216         die "type check ('number') failed - got '$value'\n";
2217     } elsif ($type eq 'string') {
2218         if (my $fmt = $confdesc->{$key}->{format}) {
2219             PVE::JSONSchema::check_format($fmt, $value);
2220             return $value;
2221         }
2222         $value =~ s/^\"(.*)\"$/$1/;
2223         return $value;
2224     } else {
2225         die "internal error"
2226     }
2227 }
2228
2229 sub destroy_vm {
2230     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2231
2232     my $conf = PVE::QemuConfig->load_config($vmid);
2233
2234     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2235         PVE::QemuConfig->check_lock($conf);
2236     }
2237
2238     if ($conf->{template}) {
2239         # check if any base image is still used by a linked clone
2240         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2241                 my ($ds, $drive) = @_;
2242                 return if drive_is_cdrom($drive);
2243
2244                 my $volid = $drive->{file};
2245                 return if !$volid || $volid =~ m|^/|;
2246
2247                 die "base volume '$volid' is still in use by linked cloned\n"
2248                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2249
2250         });
2251     }
2252
2253     my $volids = {};
2254     my $remove_owned_drive = sub {
2255         my ($ds, $drive) = @_;
2256         return if drive_is_cdrom($drive, 1);
2257
2258         my $volid = $drive->{file};
2259         return if !$volid || $volid =~ m|^/|;
2260         return if $volids->{$volid};
2261
2262         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2263         return if !$path || !$owner || ($owner != $vmid);
2264
2265         $volids->{$volid} = 1;
2266         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2267         warn "Could not remove disk '$volid', check manually: $@" if $@;
2268     };
2269
2270     # only remove disks owned by this VM (referenced in the config)
2271     my $include_opts = {
2272         include_unused => 1,
2273         extra_keys => ['vmstate'],
2274     };
2275     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2276
2277     for my $snap (values %{$conf->{snapshots}}) {
2278         next if !defined($snap->{vmstate});
2279         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2280         next if !defined($drive);
2281         $remove_owned_drive->('vmstate', $drive);
2282     }
2283
2284     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2285
2286     if ($purge_unreferenced) { # also remove unreferenced disk
2287         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2288         PVE::Storage::foreach_volid($vmdisks, sub {
2289             my ($volid, $sid, $volname, $d) = @_;
2290             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2291             warn $@ if $@;
2292         });
2293     }
2294
2295     eval { delete_ifaces_ipams_ips($conf, $vmid)};
2296     warn $@ if $@;
2297
2298     if (defined $replacement_conf) {
2299         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2300     } else {
2301         PVE::QemuConfig->destroy_config($vmid);
2302     }
2303 }
2304
2305 sub parse_vm_config {
2306     my ($filename, $raw, $strict) = @_;
2307
2308     return if !defined($raw);
2309
2310     my $res = {
2311         digest => Digest::SHA::sha1_hex($raw),
2312         snapshots => {},
2313         pending => {},
2314         cloudinit => {},
2315     };
2316
2317     my $handle_error = sub {
2318         my ($msg) = @_;
2319
2320         if ($strict) {
2321             die $msg;
2322         } else {
2323             warn $msg;
2324         }
2325     };
2326
2327     $filename =~ m|/qemu-server/(\d+)\.conf$|
2328         || die "got strange filename '$filename'";
2329
2330     my $vmid = $1;
2331
2332     my $conf = $res;
2333     my $descr;
2334     my $finish_description = sub {
2335         if (defined($descr)) {
2336             $descr =~ s/\s+$//;
2337             $conf->{description} = $descr;
2338         }
2339         $descr = undef;
2340     };
2341     my $section = '';
2342
2343     my @lines = split(/\n/, $raw);
2344     foreach my $line (@lines) {
2345         next if $line =~ m/^\s*$/;
2346
2347         if ($line =~ m/^\[PENDING\]\s*$/i) {
2348             $section = 'pending';
2349             $finish_description->();
2350             $conf = $res->{$section} = {};
2351             next;
2352         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2353             $section = 'cloudinit';
2354             $finish_description->();
2355             $conf = $res->{$section} = {};
2356             next;
2357
2358         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2359             $section = $1;
2360             $finish_description->();
2361             $conf = $res->{snapshots}->{$section} = {};
2362             next;
2363         }
2364
2365         if ($line =~ m/^\#(.*)$/) {
2366             $descr = '' if !defined($descr);
2367             $descr .= PVE::Tools::decode_text($1) . "\n";
2368             next;
2369         }
2370
2371         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2372             $descr = '' if !defined($descr);
2373             $descr .= PVE::Tools::decode_text($2);
2374         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2375             $conf->{snapstate} = $1;
2376         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2377             my $key = $1;
2378             my $value = $2;
2379             $conf->{$key} = $value;
2380         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2381             my $value = $1;
2382             if ($section eq 'pending') {
2383                 $conf->{delete} = $value; # we parse this later
2384             } else {
2385                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2386             }
2387         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2388             my $key = $1;
2389             my $value = $2;
2390             if ($section eq 'cloudinit') {
2391                 # ignore validation only used for informative purpose
2392                 $conf->{$key} = $value;
2393                 next;
2394             }
2395             eval { $value = check_type($key, $value); };
2396             if ($@) {
2397                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2398             } else {
2399                 $key = 'ide2' if $key eq 'cdrom';
2400                 my $fmt = $confdesc->{$key}->{format};
2401                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2402                     my $v = parse_drive($key, $value);
2403                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2404                         $v->{file} = $volid;
2405                         $value = print_drive($v);
2406                     } else {
2407                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2408                         next;
2409                     }
2410                 }
2411
2412                 $conf->{$key} = $value;
2413             }
2414         } else {
2415             $handle_error->("vm $vmid - unable to parse config: $line\n");
2416         }
2417     }
2418
2419     $finish_description->();
2420     delete $res->{snapstate}; # just to be sure
2421
2422     return $res;
2423 }
2424
2425 sub write_vm_config {
2426     my ($filename, $conf) = @_;
2427
2428     delete $conf->{snapstate}; # just to be sure
2429
2430     if ($conf->{cdrom}) {
2431         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2432         $conf->{ide2} = $conf->{cdrom};
2433         delete $conf->{cdrom};
2434     }
2435
2436     # we do not use 'smp' any longer
2437     if ($conf->{sockets}) {
2438         delete $conf->{smp};
2439     } elsif ($conf->{smp}) {
2440         $conf->{sockets} = $conf->{smp};
2441         delete $conf->{cores};
2442         delete $conf->{smp};
2443     }
2444
2445     my $used_volids = {};
2446
2447     my $cleanup_config = sub {
2448         my ($cref, $pending, $snapname) = @_;
2449
2450         foreach my $key (keys %$cref) {
2451             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2452                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2453             my $value = $cref->{$key};
2454             if ($key eq 'delete') {
2455                 die "propertry 'delete' is only allowed in [PENDING]\n"
2456                     if !$pending;
2457                 # fixme: check syntax?
2458                 next;
2459             }
2460             eval { $value = check_type($key, $value); };
2461             die "unable to parse value of '$key' - $@" if $@;
2462
2463             $cref->{$key} = $value;
2464
2465             if (!$snapname && is_valid_drivename($key)) {
2466                 my $drive = parse_drive($key, $value);
2467                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2468             }
2469         }
2470     };
2471
2472     &$cleanup_config($conf);
2473
2474     &$cleanup_config($conf->{pending}, 1);
2475
2476     foreach my $snapname (keys %{$conf->{snapshots}}) {
2477         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2478         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2479     }
2480
2481     # remove 'unusedX' settings if we re-add a volume
2482     foreach my $key (keys %$conf) {
2483         my $value = $conf->{$key};
2484         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2485             delete $conf->{$key};
2486         }
2487     }
2488
2489     my $generate_raw_config = sub {
2490         my ($conf, $pending) = @_;
2491
2492         my $raw = '';
2493
2494         # add description as comment to top of file
2495         if (defined(my $descr = $conf->{description})) {
2496             if ($descr) {
2497                 foreach my $cl (split(/\n/, $descr)) {
2498                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2499                 }
2500             } else {
2501                 $raw .= "#\n" if $pending;
2502             }
2503         }
2504
2505         foreach my $key (sort keys %$conf) {
2506             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2507             $raw .= "$key: $conf->{$key}\n";
2508         }
2509         return $raw;
2510     };
2511
2512     my $raw = &$generate_raw_config($conf);
2513
2514     if (scalar(keys %{$conf->{pending}})){
2515         $raw .= "\n[PENDING]\n";
2516         $raw .= &$generate_raw_config($conf->{pending}, 1);
2517     }
2518
2519     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2520         $raw .= "\n[special:cloudinit]\n";
2521         $raw .= &$generate_raw_config($conf->{cloudinit});
2522     }
2523
2524     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2525         $raw .= "\n[$snapname]\n";
2526         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2527     }
2528
2529     return $raw;
2530 }
2531
2532 sub load_defaults {
2533
2534     my $res = {};
2535
2536     # we use static defaults from our JSON schema configuration
2537     foreach my $key (keys %$confdesc) {
2538         if (defined(my $default = $confdesc->{$key}->{default})) {
2539             $res->{$key} = $default;
2540         }
2541     }
2542
2543     return $res;
2544 }
2545
2546 sub config_list {
2547     my $vmlist = PVE::Cluster::get_vmlist();
2548     my $res = {};
2549     return $res if !$vmlist || !$vmlist->{ids};
2550     my $ids = $vmlist->{ids};
2551     my $nodename = nodename();
2552
2553     foreach my $vmid (keys %$ids) {
2554         my $d = $ids->{$vmid};
2555         next if !$d->{node} || $d->{node} ne $nodename;
2556         next if !$d->{type} || $d->{type} ne 'qemu';
2557         $res->{$vmid}->{exists} = 1;
2558     }
2559     return $res;
2560 }
2561
2562 # test if VM uses local resources (to prevent migration)
2563 sub check_local_resources {
2564     my ($conf, $noerr) = @_;
2565
2566     my @loc_res = ();
2567     my $mapped_res = [];
2568
2569     my $nodelist = PVE::Cluster::get_nodelist();
2570     my $pci_map = PVE::Mapping::PCI::config();
2571     my $usb_map = PVE::Mapping::USB::config();
2572
2573     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2574
2575     my $add_missing_mapping = sub {
2576         my ($type, $key, $id) = @_;
2577         for my $node (@$nodelist) {
2578             my $entry;
2579             if ($type eq 'pci') {
2580                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2581             } elsif ($type eq 'usb') {
2582                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2583             }
2584             if (!scalar($entry->@*)) {
2585                 push @{$missing_mappings_by_node->{$node}}, $key;
2586             }
2587         }
2588     };
2589
2590     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2591     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2592
2593     push @loc_res, "ivshmem" if $conf->{ivshmem};
2594
2595     foreach my $k (keys %$conf) {
2596         if ($k =~ m/^usb/) {
2597             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2598             next if $entry->{host} =~ m/^spice$/i;
2599             if ($entry->{mapping}) {
2600                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2601                 push @$mapped_res, $k;
2602             }
2603         }
2604         if ($k =~ m/^hostpci/) {
2605             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2606             if ($entry->{mapping}) {
2607                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2608                 push @$mapped_res, $k;
2609             }
2610         }
2611         # sockets are safe: they will recreated be on the target side post-migrate
2612         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2613         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2614     }
2615
2616     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2617
2618     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2619 }
2620
2621 # check if used storages are available on all nodes (use by migrate)
2622 sub check_storage_availability {
2623     my ($storecfg, $conf, $node) = @_;
2624
2625     PVE::QemuConfig->foreach_volume($conf, sub {
2626         my ($ds, $drive) = @_;
2627
2628         my $volid = $drive->{file};
2629         return if !$volid;
2630
2631         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2632         return if !$sid;
2633
2634         # check if storage is available on both nodes
2635         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2636         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2637
2638         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2639
2640         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2641             if !$scfg->{content}->{$vtype};
2642    });
2643 }
2644
2645 # list nodes where all VM images are available (used by has_feature API)
2646 sub shared_nodes {
2647     my ($conf, $storecfg) = @_;
2648
2649     my $nodelist = PVE::Cluster::get_nodelist();
2650     my $nodehash = { map { $_ => 1 } @$nodelist };
2651     my $nodename = nodename();
2652
2653     PVE::QemuConfig->foreach_volume($conf, sub {
2654         my ($ds, $drive) = @_;
2655
2656         my $volid = $drive->{file};
2657         return if !$volid;
2658
2659         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2660         if ($storeid) {
2661             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2662             if ($scfg->{disable}) {
2663                 $nodehash = {};
2664             } elsif (my $avail = $scfg->{nodes}) {
2665                 foreach my $node (keys %$nodehash) {
2666                     delete $nodehash->{$node} if !$avail->{$node};
2667                 }
2668             } elsif (!$scfg->{shared}) {
2669                 foreach my $node (keys %$nodehash) {
2670                     delete $nodehash->{$node} if $node ne $nodename
2671                 }
2672             }
2673         }
2674     });
2675
2676     return $nodehash
2677 }
2678
2679 sub check_local_storage_availability {
2680     my ($conf, $storecfg) = @_;
2681
2682     my $nodelist = PVE::Cluster::get_nodelist();
2683     my $nodehash = { map { $_ => {} } @$nodelist };
2684
2685     PVE::QemuConfig->foreach_volume($conf, sub {
2686         my ($ds, $drive) = @_;
2687
2688         my $volid = $drive->{file};
2689         return if !$volid;
2690
2691         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2692         if ($storeid) {
2693             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2694
2695             if ($scfg->{disable}) {
2696                 foreach my $node (keys %$nodehash) {
2697                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2698                 }
2699             } elsif (my $avail = $scfg->{nodes}) {
2700                 foreach my $node (keys %$nodehash) {
2701                     if (!$avail->{$node}) {
2702                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2703                     }
2704                 }
2705             }
2706         }
2707     });
2708
2709     foreach my $node (values %$nodehash) {
2710         if (my $unavail = $node->{unavailable_storages}) {
2711             $node->{unavailable_storages} = [ sort keys %$unavail ];
2712         }
2713     }
2714
2715     return $nodehash
2716 }
2717
2718 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2719 sub check_running {
2720     my ($vmid, $nocheck, $node) = @_;
2721
2722     # $nocheck is set when called during a migration, in which case the config
2723     # file might still or already reside on the *other* node
2724     # - because rename has already happened, and current node is source
2725     # - because rename hasn't happened yet, and current node is target
2726     # - because rename has happened, current node is target, but hasn't yet
2727     # processed it yet
2728     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2729     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2730 }
2731
2732 sub vzlist {
2733
2734     my $vzlist = config_list();
2735
2736     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2737
2738     while (defined(my $de = $fd->read)) {
2739         next if $de !~ m/^(\d+)\.pid$/;
2740         my $vmid = $1;
2741         next if !defined($vzlist->{$vmid});
2742         if (my $pid = check_running($vmid)) {
2743             $vzlist->{$vmid}->{pid} = $pid;
2744         }
2745     }
2746
2747     return $vzlist;
2748 }
2749
2750 our $vmstatus_return_properties = {
2751     vmid => get_standard_option('pve-vmid'),
2752     status => {
2753         description => "QEMU process status.",
2754         type => 'string',
2755         enum => ['stopped', 'running'],
2756     },
2757     maxmem => {
2758         description => "Maximum memory in bytes.",
2759         type => 'integer',
2760         optional => 1,
2761         renderer => 'bytes',
2762     },
2763     maxdisk => {
2764         description => "Root disk size in bytes.",
2765         type => 'integer',
2766         optional => 1,
2767         renderer => 'bytes',
2768     },
2769     name => {
2770         description => "VM name.",
2771         type => 'string',
2772         optional => 1,
2773     },
2774     qmpstatus => {
2775         description => "VM run state from the 'query-status' QMP monitor command.",
2776         type => 'string',
2777         optional => 1,
2778     },
2779     pid => {
2780         description => "PID of running qemu process.",
2781         type => 'integer',
2782         optional => 1,
2783     },
2784     uptime => {
2785         description => "Uptime.",
2786         type => 'integer',
2787         optional => 1,
2788         renderer => 'duration',
2789     },
2790     cpus => {
2791         description => "Maximum usable CPUs.",
2792         type => 'number',
2793         optional => 1,
2794     },
2795     lock => {
2796         description => "The current config lock, if any.",
2797         type => 'string',
2798         optional => 1,
2799     },
2800     tags => {
2801         description  => "The current configured tags, if any",
2802         type => 'string',
2803         optional => 1,
2804     },
2805     'running-machine' => {
2806         description => "The currently running machine type (if running).",
2807         type => 'string',
2808         optional => 1,
2809     },
2810     'running-qemu' => {
2811         description => "The currently running QEMU version (if running).",
2812         type => 'string',
2813         optional => 1,
2814     },
2815 };
2816
2817 my $last_proc_pid_stat;
2818
2819 # get VM status information
2820 # This must be fast and should not block ($full == false)
2821 # We only query KVM using QMP if $full == true (this can be slow)
2822 sub vmstatus {
2823     my ($opt_vmid, $full) = @_;
2824
2825     my $res = {};
2826
2827     my $storecfg = PVE::Storage::config();
2828
2829     my $list = vzlist();
2830     my $defaults = load_defaults();
2831
2832     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2833
2834     my $cpucount = $cpuinfo->{cpus} || 1;
2835
2836     foreach my $vmid (keys %$list) {
2837         next if $opt_vmid && ($vmid ne $opt_vmid);
2838
2839         my $conf = PVE::QemuConfig->load_config($vmid);
2840
2841         my $d = { vmid => int($vmid) };
2842         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2843
2844         # fixme: better status?
2845         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2846
2847         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2848         if (defined($size)) {
2849             $d->{disk} = 0; # no info available
2850             $d->{maxdisk} = $size;
2851         } else {
2852             $d->{disk} = 0;
2853             $d->{maxdisk} = 0;
2854         }
2855
2856         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2857             * ($conf->{cores} || $defaults->{cores});
2858         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2859         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2860
2861         $d->{name} = $conf->{name} || "VM $vmid";
2862         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2863
2864         if ($conf->{balloon}) {
2865             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2866             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2867                 : $defaults->{shares};
2868         }
2869
2870         $d->{uptime} = 0;
2871         $d->{cpu} = 0;
2872         $d->{mem} = 0;
2873
2874         $d->{netout} = 0;
2875         $d->{netin} = 0;
2876
2877         $d->{diskread} = 0;
2878         $d->{diskwrite} = 0;
2879
2880         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2881
2882         $d->{serial} = 1 if conf_has_serial($conf);
2883         $d->{lock} = $conf->{lock} if $conf->{lock};
2884         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2885
2886         $res->{$vmid} = $d;
2887     }
2888
2889     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2890     foreach my $dev (keys %$netdev) {
2891         next if $dev !~ m/^tap([1-9]\d*)i/;
2892         my $vmid = $1;
2893         my $d = $res->{$vmid};
2894         next if !$d;
2895
2896         $d->{netout} += $netdev->{$dev}->{receive};
2897         $d->{netin} += $netdev->{$dev}->{transmit};
2898
2899         if ($full) {
2900             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2901             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2902         }
2903
2904     }
2905
2906     my $ctime = gettimeofday;
2907
2908     foreach my $vmid (keys %$list) {
2909
2910         my $d = $res->{$vmid};
2911         my $pid = $d->{pid};
2912         next if !$pid;
2913
2914         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2915         next if !$pstat; # not running
2916
2917         my $used = $pstat->{utime} + $pstat->{stime};
2918
2919         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2920
2921         if ($pstat->{vsize}) {
2922             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2923         }
2924
2925         my $old = $last_proc_pid_stat->{$pid};
2926         if (!$old) {
2927             $last_proc_pid_stat->{$pid} = {
2928                 time => $ctime,
2929                 used => $used,
2930                 cpu => 0,
2931             };
2932             next;
2933         }
2934
2935         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2936
2937         if ($dtime > 1000) {
2938             my $dutime = $used -  $old->{used};
2939
2940             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2941             $last_proc_pid_stat->{$pid} = {
2942                 time => $ctime,
2943                 used => $used,
2944                 cpu => $d->{cpu},
2945             };
2946         } else {
2947             $d->{cpu} = $old->{cpu};
2948         }
2949     }
2950
2951     return $res if !$full;
2952
2953     my $qmpclient = PVE::QMPClient->new();
2954
2955     my $ballooncb = sub {
2956         my ($vmid, $resp) = @_;
2957
2958         my $info = $resp->{'return'};
2959         return if !$info->{max_mem};
2960
2961         my $d = $res->{$vmid};
2962
2963         # use memory assigned to VM
2964         $d->{maxmem} = $info->{max_mem};
2965         $d->{balloon} = $info->{actual};
2966
2967         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2968             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2969             $d->{freemem} = $info->{free_mem};
2970         }
2971
2972         $d->{ballooninfo} = $info;
2973     };
2974
2975     my $blockstatscb = sub {
2976         my ($vmid, $resp) = @_;
2977         my $data = $resp->{'return'} || [];
2978         my $totalrdbytes = 0;
2979         my $totalwrbytes = 0;
2980
2981         for my $blockstat (@$data) {
2982             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2983             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2984
2985             $blockstat->{device} =~ s/drive-//;
2986             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2987         }
2988         $res->{$vmid}->{diskread} = $totalrdbytes;
2989         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2990     };
2991
2992     my $machinecb = sub {
2993         my ($vmid, $resp) = @_;
2994         my $data = $resp->{'return'} || [];
2995
2996         $res->{$vmid}->{'running-machine'} =
2997             PVE::QemuServer::Machine::current_from_query_machines($data);
2998     };
2999
3000     my $versioncb = sub {
3001         my ($vmid, $resp) = @_;
3002         my $data = $resp->{'return'} // {};
3003         my $version = 'unknown';
3004
3005         if (my $v = $data->{qemu}) {
3006             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3007         }
3008
3009         $res->{$vmid}->{'running-qemu'} = $version;
3010     };
3011
3012     my $statuscb = sub {
3013         my ($vmid, $resp) = @_;
3014
3015         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3016         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3017         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3018         # this fails if ballon driver is not loaded, so this must be
3019         # the last commnand (following command are aborted if this fails).
3020         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3021
3022         my $status = 'unknown';
3023         if (!defined($status = $resp->{'return'}->{status})) {
3024             warn "unable to get VM status\n";
3025             return;
3026         }
3027
3028         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3029     };
3030
3031     foreach my $vmid (keys %$list) {
3032         next if $opt_vmid && ($vmid ne $opt_vmid);
3033         next if !$res->{$vmid}->{pid}; # not running
3034         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3035     }
3036
3037     $qmpclient->queue_execute(undef, 2);
3038
3039     foreach my $vmid (keys %$list) {
3040         next if $opt_vmid && ($vmid ne $opt_vmid);
3041         next if !$res->{$vmid}->{pid}; #not running
3042
3043         # we can't use the $qmpclient since it might have already aborted on
3044         # 'query-balloon', but this might also fail for older versions...
3045         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3046         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3047     }
3048
3049     foreach my $vmid (keys %$list) {
3050         next if $opt_vmid && ($vmid ne $opt_vmid);
3051         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3052     }
3053
3054     return $res;
3055 }
3056
3057 sub conf_has_serial {
3058     my ($conf) = @_;
3059
3060     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3061         if ($conf->{"serial$i"}) {
3062             return 1;
3063         }
3064     }
3065
3066     return 0;
3067 }
3068
3069 sub conf_has_audio {
3070     my ($conf, $id) = @_;
3071
3072     $id //= 0;
3073     my $audio = $conf->{"audio$id"};
3074     return if !defined($audio);
3075
3076     my $audioproperties = parse_property_string($audio_fmt, $audio);
3077     my $audiodriver = $audioproperties->{driver} // 'spice';
3078
3079     return {
3080         dev => $audioproperties->{device},
3081         dev_id => "audiodev$id",
3082         backend => $audiodriver,
3083         backend_id => "$audiodriver-backend${id}",
3084     };
3085 }
3086
3087 sub audio_devs {
3088     my ($audio, $audiopciaddr, $machine_version) = @_;
3089
3090     my $devs = [];
3091
3092     my $id = $audio->{dev_id};
3093     my $audiodev = "";
3094     if (min_version($machine_version, 4, 2)) {
3095         $audiodev = ",audiodev=$audio->{backend_id}";
3096     }
3097
3098     if ($audio->{dev} eq 'AC97') {
3099         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3100     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3101         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3102         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3103         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3104     } else {
3105         die "unkown audio device '$audio->{dev}', implement me!";
3106     }
3107
3108     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3109
3110     return $devs;
3111 }
3112
3113 sub get_tpm_paths {
3114     my ($vmid) = @_;
3115     return {
3116         socket => "/var/run/qemu-server/$vmid.swtpm",
3117         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3118     };
3119 }
3120
3121 sub add_tpm_device {
3122     my ($vmid, $devices, $conf) = @_;
3123
3124     return if !$conf->{tpmstate0};
3125
3126     my $paths = get_tpm_paths($vmid);
3127
3128     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3129     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3130     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3131 }
3132
3133 sub start_swtpm {
3134     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3135
3136     return if !$tpmdrive;
3137
3138     my $state;
3139     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3140     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3141     if ($storeid) {
3142         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3143     } else {
3144         $state = $tpm->{file};
3145     }
3146
3147     my $paths = get_tpm_paths($vmid);
3148
3149     # during migration, we will get state from remote
3150     #
3151     if (!$migration) {
3152         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3153         my $setup_cmd = [
3154             "swtpm_setup",
3155             "--tpmstate",
3156             "file://$state",
3157             "--createek",
3158             "--create-ek-cert",
3159             "--create-platform-cert",
3160             "--lock-nvram",
3161             "--config",
3162             "/etc/swtpm_setup.conf", # do not use XDG configs
3163             "--runas",
3164             "0", # force creation as root, error if not possible
3165             "--not-overwrite", # ignore existing state, do not modify
3166         ];
3167
3168         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3169         # TPM 2.0 supports ECC crypto, use if possible
3170         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3171
3172         run_command($setup_cmd, outfunc => sub {
3173             print "swtpm_setup: $1\n";
3174         });
3175     }
3176
3177     # Used to distinguish different invocations in the log.
3178     my $log_prefix = "[id=" . int(time()) . "] ";
3179
3180     my $emulator_cmd = [
3181         "swtpm",
3182         "socket",
3183         "--tpmstate",
3184         "backend-uri=file://$state,mode=0600",
3185         "--ctrl",
3186         "type=unixio,path=$paths->{socket},mode=0600",
3187         "--pid",
3188         "file=$paths->{pid}",
3189         "--terminate", # terminate on QEMU disconnect
3190         "--daemon",
3191         "--log",
3192         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3193     ];
3194     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3195     run_command($emulator_cmd, outfunc => sub { print $1; });
3196
3197     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3198     while (! -e $paths->{pid}) {
3199         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3200         usleep(50_000);
3201     }
3202
3203     # return untainted PID of swtpm daemon so it can be killed on error
3204     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3205     return $1;
3206 }
3207
3208 sub vga_conf_has_spice {
3209     my ($vga) = @_;
3210
3211     my $vgaconf = parse_vga($vga);
3212     my $vgatype = $vgaconf->{type};
3213     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3214
3215     return $1 || 1;
3216 }
3217
3218 sub get_vm_arch {
3219     my ($conf) = @_;
3220     return $conf->{arch} // get_host_arch();
3221 }
3222
3223 my $default_machines = {
3224     x86_64 => 'pc',
3225     aarch64 => 'virt',
3226 };
3227
3228 sub get_installed_machine_version {
3229     my ($kvmversion) = @_;
3230     $kvmversion = kvm_user_version() if !defined($kvmversion);
3231     $kvmversion =~ m/^(\d+\.\d+)/;
3232     return $1;
3233 }
3234
3235 sub windows_get_pinned_machine_version {
3236     my ($machine, $base_version, $kvmversion) = @_;
3237
3238     my $pin_version = $base_version;
3239     if (!defined($base_version) ||
3240         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3241     ) {
3242         $pin_version = get_installed_machine_version($kvmversion);
3243     }
3244     if (!$machine || $machine eq 'pc') {
3245         $machine = "pc-i440fx-$pin_version";
3246     } elsif ($machine eq 'q35') {
3247         $machine = "pc-q35-$pin_version";
3248     } elsif ($machine eq 'virt') {
3249         $machine = "virt-$pin_version";
3250     } else {
3251         warn "unknown machine type '$machine', not touching that!\n";
3252     }
3253
3254     return $machine;
3255 }
3256
3257 sub get_vm_machine {
3258     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3259
3260     my $machine = $forcemachine || $conf->{machine};
3261
3262     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3263         $kvmversion //= kvm_user_version();
3264         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3265         # layout which confuses windows quite a bit and may result in various regressions..
3266         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3267         if (windows_version($conf->{ostype})) {
3268             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3269         }
3270         $arch //= 'x86_64';
3271         $machine ||= $default_machines->{$arch};
3272         if ($add_pve_version) {
3273             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3274             $machine .= "+pve$pvever";
3275         }
3276     }
3277
3278     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3279         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3280         $machine = $1 if $is_pxe;
3281
3282         # for version-pinned machines that do not include a pve-version (e.g.
3283         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3284         $machine .= '+pve0';
3285
3286         $machine .= '.pxe' if $is_pxe;
3287     }
3288
3289     return $machine;
3290 }
3291
3292 sub get_ovmf_files($$$) {
3293     my ($arch, $efidisk, $smm) = @_;
3294
3295     my $types = $OVMF->{$arch}
3296         or die "no OVMF images known for architecture '$arch'\n";
3297
3298     my $type = 'default';
3299     if ($arch eq 'x86_64') {
3300         if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3301             $type = $smm ? "4m" : "4m-no-smm";
3302             $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3303         } else {
3304             # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3305         }
3306     }
3307
3308     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3309     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3310     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3311
3312     return ($ovmf_code, $ovmf_vars);
3313 }
3314
3315 my $Arch2Qemu = {
3316     aarch64 => '/usr/bin/qemu-system-aarch64',
3317     x86_64 => '/usr/bin/qemu-system-x86_64',
3318 };
3319 sub get_command_for_arch($) {
3320     my ($arch) = @_;
3321     return '/usr/bin/kvm' if is_native_arch($arch);
3322
3323     my $cmd = $Arch2Qemu->{$arch}
3324         or die "don't know how to emulate architecture '$arch'\n";
3325     return $cmd;
3326 }
3327
3328 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3329 # to use in a QEMU command line (-cpu element), first array_intersect the result
3330 # of query_supported_ with query_understood_. This is necessary because:
3331 #
3332 # a) query_understood_ returns flags the host cannot use and
3333 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3334 #    flags, but CPU settings - with most of them being flags. Those settings
3335 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3336 #
3337 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3338 # expensive. If you need the value returned from this, you can get it much
3339 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3340 # $accel being 'kvm' or 'tcg'.
3341 #
3342 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3343 # changes, automatically populating pmxcfs.
3344 #
3345 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3346 # since kvm and tcg machines support different flags
3347 #
3348 sub query_supported_cpu_flags {
3349     my ($arch) = @_;
3350
3351     $arch //= get_host_arch();
3352     my $default_machine = $default_machines->{$arch};
3353
3354     my $flags = {};
3355
3356     # FIXME: Once this is merged, the code below should work for ARM as well:
3357     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3358     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3359         $arch eq "aarch64";
3360
3361     my $kvm_supported = defined(kvm_version());
3362     my $qemu_cmd = get_command_for_arch($arch);
3363     my $fakevmid = -1;
3364     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3365
3366     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3367     my $query_supported_run_qemu = sub {
3368         my ($kvm) = @_;
3369
3370         my $flags = {};
3371         my $cmd = [
3372             $qemu_cmd,
3373             '-machine', $default_machine,
3374             '-display', 'none',
3375             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3376             '-mon', 'chardev=qmp,mode=control',
3377             '-pidfile', $pidfile,
3378             '-S', '-daemonize'
3379         ];
3380
3381         if (!$kvm) {
3382             push @$cmd, '-accel', 'tcg';
3383         }
3384
3385         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3386         die "QEMU flag querying VM exited with code " . $rc if $rc;
3387
3388         eval {
3389             my $cmd_result = mon_cmd(
3390                 $fakevmid,
3391                 'query-cpu-model-expansion',
3392                 type => 'full',
3393                 model => { name => 'host' }
3394             );
3395
3396             my $props = $cmd_result->{model}->{props};
3397             foreach my $prop (keys %$props) {
3398                 next if $props->{$prop} ne '1';
3399                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3400                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3401                 # We only keep those with underscores, to match /proc/cpuinfo
3402                 $prop =~ s/\.|-/_/g;
3403                 $flags->{$prop} = 1;
3404             }
3405         };
3406         my $err = $@;
3407
3408         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3409         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3410
3411         die $err if $err;
3412
3413         return [ sort keys %$flags ];
3414     };
3415
3416     # We need to query QEMU twice, since KVM and TCG have different supported flags
3417     PVE::QemuConfig->lock_config($fakevmid, sub {
3418         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3419         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3420
3421         if ($kvm_supported) {
3422             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3423             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3424         }
3425     });
3426
3427     return $flags;
3428 }
3429
3430 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3431 my $understood_cpu_flag_dir = "/usr/share/kvm";
3432 sub query_understood_cpu_flags {
3433     my $arch = get_host_arch();
3434     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3435
3436     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3437         if ! -e $filepath;
3438
3439     my $raw = file_get_contents($filepath);
3440     $raw =~ s/^\s+|\s+$//g;
3441     my @flags = split(/\s+/, $raw);
3442
3443     return \@flags;
3444 }
3445
3446 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3447 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3448 my sub should_disable_smm {
3449     my ($conf, $vga, $machine) = @_;
3450
3451     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3452
3453     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3454         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3455 }
3456
3457 my sub print_ovmf_drive_commandlines {
3458     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3459
3460     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3461
3462     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3463
3464     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3465     if ($d) {
3466         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3467         my ($path, $format) = $d->@{'file', 'format'};
3468         if ($storeid) {
3469             $path = PVE::Storage::path($storecfg, $d->{file});
3470             if (!defined($format)) {
3471                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3472                 $format = qemu_img_format($scfg, $volname);
3473             }
3474         } elsif (!defined($format)) {
3475             die "efidisk format must be specified\n";
3476         }
3477         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3478         if ($path =~ m/^rbd:/) {
3479             $var_drive_str .= ',cache=writeback';
3480             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3481         }
3482         $var_drive_str .= ",format=$format,file=$path";
3483
3484         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3485         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3486     } else {
3487         log_warn("no efidisk configured! Using temporary efivars disk.");
3488         my $path = "/tmp/$vmid-ovmf.fd";
3489         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3490         $var_drive_str .= ",format=raw,file=$path";
3491         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3492     }
3493
3494     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3495 }
3496
3497 sub config_to_command {
3498     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3499         $pbs_backing) = @_;
3500
3501     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3502     my $devices = [];
3503     my $bridges = {};
3504     my $ostype = $conf->{ostype};
3505     my $winversion = windows_version($ostype);
3506     my $kvm = $conf->{kvm};
3507     my $nodename = nodename();
3508
3509     my $arch = get_vm_arch($conf);
3510     my $kvm_binary = get_command_for_arch($arch);
3511     my $kvmver = kvm_user_version($kvm_binary);
3512
3513     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3514         $kvmver //= "undefined";
3515         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3516     }
3517
3518     my $add_pve_version = min_version($kvmver, 4, 1);
3519
3520     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3521     my $machine_version = extract_version($machine_type, $kvmver);
3522     $kvm //= 1 if is_native_arch($arch);
3523
3524     $machine_version =~ m/(\d+)\.(\d+)/;
3525     my ($machine_major, $machine_minor) = ($1, $2);
3526
3527     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3528         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3529     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3530         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3531             ." please upgrade node '$nodename'\n"
3532     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3533         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3534         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3535             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3536             ." node '$nodename'\n";
3537     }
3538
3539     # if a specific +pve version is required for a feature, use $version_guard
3540     # instead of min_version to allow machines to be run with the minimum
3541     # required version
3542     my $required_pve_version = 0;
3543     my $version_guard = sub {
3544         my ($major, $minor, $pve) = @_;
3545         return 0 if !min_version($machine_version, $major, $minor, $pve);
3546         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3547         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3548         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3549         return 1;
3550     };
3551
3552     if ($kvm && !defined kvm_version()) {
3553         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3554             ." or enable in BIOS.\n";
3555     }
3556
3557     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3558     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3559     my $use_old_bios_files = undef;
3560     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3561
3562     my $cmd = [];
3563     if ($conf->{affinity}) {
3564         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3565     }
3566
3567     push @$cmd, $kvm_binary;
3568
3569     push @$cmd, '-id', $vmid;
3570
3571     my $vmname = $conf->{name} || "vm$vmid";
3572
3573     push @$cmd, '-name', "$vmname,debug-threads=on";
3574
3575     push @$cmd, '-no-shutdown';
3576
3577     my $use_virtio = 0;
3578
3579     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3580     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3581     push @$cmd, '-mon', "chardev=qmp,mode=control";
3582
3583     if (min_version($machine_version, 2, 12)) {
3584         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3585         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3586     }
3587
3588     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3589
3590     push @$cmd, '-daemonize';
3591
3592     if ($conf->{smbios1}) {
3593         my $smbios_conf = parse_smbios1($conf->{smbios1});
3594         if ($smbios_conf->{base64}) {
3595             # Do not pass base64 flag to qemu
3596             delete $smbios_conf->{base64};
3597             my $smbios_string = "";
3598             foreach my $key (keys %$smbios_conf) {
3599                 my $value;
3600                 if ($key eq "uuid") {
3601                     $value = $smbios_conf->{uuid}
3602                 } else {
3603                     $value = decode_base64($smbios_conf->{$key});
3604                 }
3605                 # qemu accepts any binary data, only commas need escaping by double comma
3606                 $value =~ s/,/,,/g;
3607                 $smbios_string .= "," . $key . "=" . $value if $value;
3608             }
3609             push @$cmd, '-smbios', "type=1" . $smbios_string;
3610         } else {
3611             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3612         }
3613     }
3614
3615     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3616         die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n"
3617             if !$forcecpu && get_cpu_bitness($conf->{cpu}, $arch) == 32;
3618
3619         my ($code_drive_str, $var_drive_str) =
3620             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3621         push $cmd->@*, '-drive', $code_drive_str;
3622         push $cmd->@*, '-drive', $var_drive_str;
3623     }
3624
3625     if ($q35) { # tell QEMU to load q35 config early
3626         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3627         if (min_version($machine_version, 4, 0)) {
3628             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3629         } else {
3630             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3631         }
3632     }
3633
3634     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3635         push @$cmd, $fixups->@*;
3636     }
3637
3638     if ($conf->{vmgenid}) {
3639         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3640     }
3641
3642     # add usb controllers
3643     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3644         $conf, $bridges, $arch, $machine_type, $machine_version);
3645     push @$devices, @usbcontrollers if @usbcontrollers;
3646     my $vga = parse_vga($conf->{vga});
3647
3648     my $qxlnum = vga_conf_has_spice($conf->{vga});
3649     $vga->{type} = 'qxl' if $qxlnum;
3650
3651     if (!$vga->{type}) {
3652         if ($arch eq 'aarch64') {
3653             $vga->{type} = 'virtio';
3654         } elsif (min_version($machine_version, 2, 9)) {
3655             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3656         } else {
3657             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3658         }
3659     }
3660
3661     # enable absolute mouse coordinates (needed by vnc)
3662     my $tablet = $conf->{tablet};
3663     if (!defined($tablet)) {
3664         $tablet = $defaults->{tablet};
3665         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3666         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3667     }
3668
3669     if ($tablet) {
3670         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3671         my $kbd = print_keyboarddevice_full($conf, $arch);
3672         push @$devices, '-device', $kbd if defined($kbd);
3673     }
3674
3675     my $bootorder = device_bootorder($conf);
3676
3677     # host pci device passthrough
3678     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3679         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3680
3681     # usb devices
3682     my $usb_dev_features = {};
3683     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3684
3685     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3686         $conf, $usb_dev_features, $bootorder, $machine_version);
3687     push @$devices, @usbdevices if @usbdevices;
3688
3689     # serial devices
3690     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3691         my $path = $conf->{"serial$i"} or next;
3692         if ($path eq 'socket') {
3693             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3694             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3695             # On aarch64, serial0 is the UART device. QEMU only allows
3696             # connecting UART devices via the '-serial' command line, as
3697             # the device has a fixed slot on the hardware...
3698             if ($arch eq 'aarch64' && $i == 0) {
3699                 push @$devices, '-serial', "chardev:serial$i";
3700             } else {
3701                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3702             }
3703         } else {
3704             die "no such serial device\n" if ! -c $path;
3705             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3706             push @$devices, '-device', "isa-serial,chardev=serial$i";
3707         }
3708     }
3709
3710     # parallel devices
3711     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3712         if (my $path = $conf->{"parallel$i"}) {
3713             die "no such parallel device\n" if ! -c $path;
3714             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3715             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3716             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3717         }
3718     }
3719
3720     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3721         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3722         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3723         push @$devices, @$audio_devs;
3724     }
3725
3726     # Add a TPM only if the VM is not a template,
3727     # to support backing up template VMs even if the TPM disk is write-protected.
3728     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3729
3730     my $sockets = 1;
3731     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3732     $sockets = $conf->{sockets} if  $conf->{sockets};
3733
3734     my $cores = $conf->{cores} || 1;
3735
3736     my $maxcpus = $sockets * $cores;
3737
3738     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3739
3740     my $allowed_vcpus = $cpuinfo->{cpus};
3741
3742     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3743
3744     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3745         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3746         for (my $i = 2; $i <= $vcpus; $i++)  {
3747             my $cpustr = print_cpu_device($conf, $arch, $i);
3748             push @$cmd, '-device', $cpustr;
3749         }
3750
3751     } else {
3752
3753         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3754     }
3755     push @$cmd, '-nodefaults';
3756
3757     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3758
3759     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3760
3761     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3762
3763     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3764         push @$devices, '-device', print_vga_device(
3765             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3766
3767         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3768
3769         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3770         push @$cmd,  '-vnc', "unix:$socket,password=on";
3771     } else {
3772         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3773         push @$cmd, '-nographic';
3774     }
3775
3776     # time drift fix
3777     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3778     my $useLocaltime = $conf->{localtime};
3779
3780     if ($winversion >= 5) { # windows
3781         $useLocaltime = 1 if !defined($conf->{localtime});
3782
3783         # use time drift fix when acpi is enabled
3784         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3785             $tdf = 1 if !defined($conf->{tdf});
3786         }
3787     }
3788
3789     if ($winversion >= 6) {
3790         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3791         push @$machineFlags, 'hpet=off';
3792     }
3793
3794     push @$rtcFlags, 'driftfix=slew' if $tdf;
3795
3796     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3797         push @$rtcFlags, "base=$conf->{startdate}";
3798     } elsif ($useLocaltime) {
3799         push @$rtcFlags, 'base=localtime';
3800     }
3801
3802     if ($forcecpu) {
3803         push @$cmd, '-cpu', $forcecpu;
3804     } else {
3805         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3806     }
3807
3808     PVE::QemuServer::Memory::config(
3809         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3810
3811     push @$cmd, '-S' if $conf->{freeze};
3812
3813     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3814
3815     my $guest_agent = parse_guest_agent($conf);
3816
3817     if ($guest_agent->{enabled}) {
3818         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3819         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3820
3821         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3822             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3823             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3824             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3825         } elsif ($guest_agent->{type} eq 'isa') {
3826             push @$devices, '-device', "isa-serial,chardev=qga0";
3827         }
3828     }
3829
3830     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3831     if ($rng && $version_guard->(4, 1, 2)) {
3832         check_rng_source($rng->{source});
3833
3834         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3835         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3836         my $limiter_str = "";
3837         if ($max_bytes) {
3838             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3839         }
3840
3841         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3842         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3843         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3844     }
3845
3846     my $spice_port;
3847
3848     assert_clipboard_config($vga);
3849     my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3850
3851     if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3852         if ($qxlnum > 1) {
3853             if ($winversion){
3854                 for (my $i = 1; $i < $qxlnum; $i++){
3855                     push @$devices, '-device', print_vga_device(
3856                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3857                 }
3858             } else {
3859                 # assume other OS works like Linux
3860                 my ($ram, $vram) = ("134217728", "67108864");
3861                 if ($vga->{memory}) {
3862                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3863                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3864                 }
3865                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3866                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3867             }
3868         }
3869
3870         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3871
3872         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3873         if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3874             push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3875         } else {
3876             push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3877         }
3878         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3879
3880         if ($is_spice) {
3881             my $pfamily = PVE::Tools::get_host_address_family($nodename);
3882             my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3883             die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3884
3885             my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3886             $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3887
3888             my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3889             my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3890             if ($spice_enhancement->{foldersharing}) {
3891                 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3892                 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3893             }
3894
3895             my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3896             $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3897                 if $spice_enhancement->{videostreaming};
3898             push @$devices, '-spice', "$spice_opts";
3899         }
3900     }
3901
3902     # enable balloon by default, unless explicitly disabled
3903     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3904         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3905         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3906         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3907         push @$devices, '-device', $ballooncmd;
3908     }
3909
3910     if ($conf->{watchdog}) {
3911         my $wdopts = parse_watchdog($conf->{watchdog});
3912         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3913         my $watchdog = $wdopts->{model} || 'i6300esb';
3914         push @$devices, '-device', "$watchdog$pciaddr";
3915         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3916     }
3917
3918     my $vollist = [];
3919     my $scsicontroller = {};
3920     my $ahcicontroller = {};
3921     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3922
3923     # Add iscsi initiator name if available
3924     if (my $initiator = get_initiator_name()) {
3925         push @$devices, '-iscsi', "initiator-name=$initiator";
3926     }
3927
3928     PVE::QemuConfig->foreach_volume($conf, sub {
3929         my ($ds, $drive) = @_;
3930
3931         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3932             check_volume_storage_type($storecfg, $drive->{file});
3933             push @$vollist, $drive->{file};
3934         }
3935
3936         # ignore efidisk here, already added in bios/fw handling code above
3937         return if $drive->{interface} eq 'efidisk';
3938         # similar for TPM
3939         return if $drive->{interface} eq 'tpmstate';
3940
3941         $use_virtio = 1 if $ds =~ m/^virtio/;
3942
3943         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3944
3945         if ($drive->{interface} eq 'virtio'){
3946            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3947         }
3948
3949         if ($drive->{interface} eq 'scsi') {
3950
3951             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3952
3953             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3954                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3955
3956             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3957             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3958
3959             my $iothread = '';
3960             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3961                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3962                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3963             } elsif ($drive->{iothread}) {
3964                 log_warn(
3965                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3966                 );
3967             }
3968
3969             my $queues = '';
3970             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3971                 $queues = ",num_queues=$drive->{queues}";
3972             }
3973
3974             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3975                 if !$scsicontroller->{$controller};
3976             $scsicontroller->{$controller}=1;
3977         }
3978
3979         if ($drive->{interface} eq 'sata') {
3980             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3981             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3982             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3983                 if !$ahcicontroller->{$controller};
3984             $ahcicontroller->{$controller}=1;
3985         }
3986
3987         my $pbs_conf = $pbs_backing->{$ds};
3988         my $pbs_name = undef;
3989         if ($pbs_conf) {
3990             $pbs_name = "drive-$ds-pbs";
3991             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3992         }
3993
3994         my $drive_cmd = print_drive_commandline_full(
3995             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3996
3997         # extra protection for templates, but SATA and IDE don't support it..
3998         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3999
4000         push @$devices, '-drive',$drive_cmd;
4001         push @$devices, '-device', print_drivedevice_full(
4002             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4003     });
4004
4005     for (my $i = 0; $i < $MAX_NETS; $i++) {
4006         my $netname = "net$i";
4007
4008         next if !$conf->{$netname};
4009         my $d = parse_net($conf->{$netname});
4010         next if !$d;
4011         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4012
4013         $use_virtio = 1 if $d->{model} eq 'virtio';
4014
4015         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4016
4017         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4018         push @$devices, '-netdev', $netdevfull;
4019
4020         my $netdevicefull = print_netdevice_full(
4021             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4022
4023         push @$devices, '-device', $netdevicefull;
4024     }
4025
4026     if ($conf->{ivshmem}) {
4027         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4028
4029         my $bus;
4030         if ($q35) {
4031             $bus = print_pcie_addr("ivshmem");
4032         } else {
4033             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4034         }
4035
4036         my $ivshmem_name = $ivshmem->{name} // $vmid;
4037         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4038
4039         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4040         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4041             .",size=$ivshmem->{size}M";
4042     }
4043
4044     # pci.4 is nested in pci.1
4045     $bridges->{1} = 1 if $bridges->{4};
4046
4047     if (!$q35) { # add pci bridges
4048         if (min_version($machine_version, 2, 3)) {
4049            $bridges->{1} = 1;
4050            $bridges->{2} = 1;
4051         }
4052         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4053     }
4054
4055     for my $k (sort {$b cmp $a} keys %$bridges) {
4056         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4057
4058         my $k_name = $k;
4059         if ($k == 2 && $legacy_igd) {
4060             $k_name = "$k-igd";
4061         }
4062         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4063         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4064
4065         if ($q35) { # add after -readconfig pve-q35.cfg
4066             splice @$devices, 2, 0, '-device', $devstr;
4067         } else {
4068             unshift @$devices, '-device', $devstr if $k > 0;
4069         }
4070     }
4071
4072     if (!$kvm) {
4073         push @$machineFlags, 'accel=tcg';
4074     }
4075
4076     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4077
4078     my $machine_type_min = $machine_type;
4079     if ($add_pve_version) {
4080         $machine_type_min =~ s/\+pve\d+$//;
4081         $machine_type_min .= "+pve$required_pve_version";
4082     }
4083     push @$machineFlags, "type=${machine_type_min}";
4084
4085     push @$cmd, @$devices;
4086     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4087     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4088     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4089
4090     if (my $vmstate = $conf->{vmstate}) {
4091         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4092         push @$vollist, $vmstate;
4093         push @$cmd, '-loadstate', $statepath;
4094         print "activating and using '$vmstate' as vmstate\n";
4095     }
4096
4097     if (PVE::QemuConfig->is_template($conf)) {
4098         # needed to workaround base volumes being read-only
4099         push @$cmd, '-snapshot';
4100     }
4101
4102     # add custom args
4103     if ($conf->{args}) {
4104         my $aa = PVE::Tools::split_args($conf->{args});
4105         push @$cmd, @$aa;
4106     }
4107
4108     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4109 }
4110
4111 sub check_rng_source {
4112     my ($source) = @_;
4113
4114     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4115     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4116         if ! -e $source;
4117
4118     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4119     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4120         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4121         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4122         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4123             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4124             ." to the host.\n";
4125     }
4126 }
4127
4128 sub spice_port {
4129     my ($vmid) = @_;
4130
4131     my $res = mon_cmd($vmid, 'query-spice');
4132
4133     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4134 }
4135
4136 sub vm_devices_list {
4137     my ($vmid) = @_;
4138
4139     my $res = mon_cmd($vmid, 'query-pci');
4140     my $devices_to_check = [];
4141     my $devices = {};
4142     foreach my $pcibus (@$res) {
4143         push @$devices_to_check, @{$pcibus->{devices}},
4144     }
4145
4146     while (@$devices_to_check) {
4147         my $to_check = [];
4148         for my $d (@$devices_to_check) {
4149             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4150             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4151
4152             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4153             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4154         }
4155         $devices_to_check = $to_check;
4156     }
4157
4158     my $resblock = mon_cmd($vmid, 'query-block');
4159     foreach my $block (@$resblock) {
4160         if($block->{device} =~ m/^drive-(\S+)/){
4161                 $devices->{$1} = 1;
4162         }
4163     }
4164
4165     my $resmice = mon_cmd($vmid, 'query-mice');
4166     foreach my $mice (@$resmice) {
4167         if ($mice->{name} eq 'QEMU HID Tablet') {
4168             $devices->{tablet} = 1;
4169             last;
4170         }
4171     }
4172
4173     # for usb devices there is no query-usb
4174     # but we can iterate over the entries in
4175     # qom-list path=/machine/peripheral
4176     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4177     foreach my $per (@$resperipheral) {
4178         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4179             $devices->{$per->{name}} = 1;
4180         }
4181     }
4182
4183     return $devices;
4184 }
4185
4186 sub vm_deviceplug {
4187     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4188
4189     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4190
4191     my $devices_list = vm_devices_list($vmid);
4192     return 1 if defined($devices_list->{$deviceid});
4193
4194     # add PCI bridge if we need it for the device
4195     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4196
4197     if ($deviceid eq 'tablet') {
4198         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4199     } elsif ($deviceid eq 'keyboard') {
4200         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4201     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4202         my $id = $1;
4203         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4204         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4205     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4206         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4207     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4208         qemu_iothread_add($vmid, $deviceid, $device);
4209
4210         qemu_driveadd($storecfg, $vmid, $device);
4211         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4212
4213         qemu_deviceadd($vmid, $devicefull);
4214         eval { qemu_deviceaddverify($vmid, $deviceid); };
4215         if (my $err = $@) {
4216             eval { qemu_drivedel($vmid, $deviceid); };
4217             warn $@ if $@;
4218             die $err;
4219         }
4220     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4221         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4222         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4223         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4224
4225         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4226
4227         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4228             qemu_iothread_add($vmid, $deviceid, $device);
4229             $devicefull .= ",iothread=iothread-$deviceid";
4230         }
4231
4232         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4233             $devicefull .= ",num_queues=$device->{queues}";
4234         }
4235
4236         qemu_deviceadd($vmid, $devicefull);
4237         qemu_deviceaddverify($vmid, $deviceid);
4238     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4239         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4240         qemu_driveadd($storecfg, $vmid, $device);
4241
4242         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4243         eval { qemu_deviceadd($vmid, $devicefull); };
4244         if (my $err = $@) {
4245             eval { qemu_drivedel($vmid, $deviceid); };
4246             warn $@ if $@;
4247             die $err;
4248         }
4249     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4250         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4251
4252         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4253         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4254         my $use_old_bios_files = undef;
4255         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4256
4257         my $netdevicefull = print_netdevice_full(
4258             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4259         qemu_deviceadd($vmid, $netdevicefull);
4260         eval {
4261             qemu_deviceaddverify($vmid, $deviceid);
4262             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4263         };
4264         if (my $err = $@) {
4265             eval { qemu_netdevdel($vmid, $deviceid); };
4266             warn $@ if $@;
4267             die $err;
4268         }
4269     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4270         my $bridgeid = $2;
4271         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4272         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4273
4274         qemu_deviceadd($vmid, $devicefull);
4275         qemu_deviceaddverify($vmid, $deviceid);
4276     } else {
4277         die "can't hotplug device '$deviceid'\n";
4278     }
4279
4280     return 1;
4281 }
4282
4283 # fixme: this should raise exceptions on error!
4284 sub vm_deviceunplug {
4285     my ($vmid, $conf, $deviceid) = @_;
4286
4287     my $devices_list = vm_devices_list($vmid);
4288     return 1 if !defined($devices_list->{$deviceid});
4289
4290     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4291     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4292
4293     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4294         qemu_devicedel($vmid, $deviceid);
4295     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4296         qemu_devicedel($vmid, $deviceid);
4297         qemu_devicedelverify($vmid, $deviceid);
4298     } elsif ($deviceid =~ m/^usb\d+$/) {
4299         qemu_devicedel($vmid, $deviceid);
4300         qemu_devicedelverify($vmid, $deviceid);
4301     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4302         my $device = parse_drive($deviceid, $conf->{$deviceid});
4303
4304         qemu_devicedel($vmid, $deviceid);
4305         qemu_devicedelverify($vmid, $deviceid);
4306         qemu_drivedel($vmid, $deviceid);
4307         qemu_iothread_del($vmid, $deviceid, $device);
4308     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4309         qemu_devicedel($vmid, $deviceid);
4310         qemu_devicedelverify($vmid, $deviceid);
4311     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4312         my $device = parse_drive($deviceid, $conf->{$deviceid});
4313
4314         qemu_devicedel($vmid, $deviceid);
4315         qemu_devicedelverify($vmid, $deviceid);
4316         qemu_drivedel($vmid, $deviceid);
4317         qemu_deletescsihw($conf, $vmid, $deviceid);
4318
4319         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4320             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4321     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4322         qemu_devicedel($vmid, $deviceid);
4323         qemu_devicedelverify($vmid, $deviceid);
4324         qemu_netdevdel($vmid, $deviceid);
4325     } else {
4326         die "can't unplug device '$deviceid'\n";
4327     }
4328
4329     return 1;
4330 }
4331
4332 sub qemu_spice_usbredir_chardev_add {
4333     my ($vmid, $id) = @_;
4334
4335     mon_cmd($vmid, "chardev-add" , (
4336         id => $id,
4337         backend => {
4338             type => 'spicevmc',
4339             data => {
4340                 type => "usbredir",
4341             },
4342         },
4343     ));
4344 }
4345
4346 sub qemu_iothread_add {
4347     my ($vmid, $deviceid, $device) = @_;
4348
4349     if ($device->{iothread}) {
4350         my $iothreads = vm_iothreads_list($vmid);
4351         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4352     }
4353 }
4354
4355 sub qemu_iothread_del {
4356     my ($vmid, $deviceid, $device) = @_;
4357
4358     if ($device->{iothread}) {
4359         my $iothreads = vm_iothreads_list($vmid);
4360         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4361     }
4362 }
4363
4364 sub qemu_driveadd {
4365     my ($storecfg, $vmid, $device) = @_;
4366
4367     my $kvmver = get_running_qemu_version($vmid);
4368     my $io_uring = min_version($kvmver, 6, 0);
4369     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4370     $drive =~ s/\\/\\\\/g;
4371     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4372
4373     # If the command succeeds qemu prints: "OK"
4374     return 1 if $ret =~ m/OK/s;
4375
4376     die "adding drive failed: $ret\n";
4377 }
4378
4379 sub qemu_drivedel {
4380     my ($vmid, $deviceid) = @_;
4381
4382     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4383     $ret =~ s/^\s+//;
4384
4385     return 1 if $ret eq "";
4386
4387     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4388     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4389
4390     die "deleting drive $deviceid failed : $ret\n";
4391 }
4392
4393 sub qemu_deviceaddverify {
4394     my ($vmid, $deviceid) = @_;
4395
4396     for (my $i = 0; $i <= 5; $i++) {
4397          my $devices_list = vm_devices_list($vmid);
4398          return 1 if defined($devices_list->{$deviceid});
4399          sleep 1;
4400     }
4401
4402     die "error on hotplug device '$deviceid'\n";
4403 }
4404
4405
4406 sub qemu_devicedelverify {
4407     my ($vmid, $deviceid) = @_;
4408
4409     # need to verify that the device is correctly removed as device_del
4410     # is async and empty return is not reliable
4411
4412     for (my $i = 0; $i <= 5; $i++) {
4413          my $devices_list = vm_devices_list($vmid);
4414          return 1 if !defined($devices_list->{$deviceid});
4415          sleep 1;
4416     }
4417
4418     die "error on hot-unplugging device '$deviceid'\n";
4419 }
4420
4421 sub qemu_findorcreatescsihw {
4422     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4423
4424     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4425
4426     my $scsihwid="$controller_prefix$controller";
4427     my $devices_list = vm_devices_list($vmid);
4428
4429     if (!defined($devices_list->{$scsihwid})) {
4430         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4431     }
4432
4433     return 1;
4434 }
4435
4436 sub qemu_deletescsihw {
4437     my ($conf, $vmid, $opt) = @_;
4438
4439     my $device = parse_drive($opt, $conf->{$opt});
4440
4441     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4442         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4443         return 1;
4444     }
4445
4446     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4447
4448     my $devices_list = vm_devices_list($vmid);
4449     foreach my $opt (keys %{$devices_list}) {
4450         if (is_valid_drivename($opt)) {
4451             my $drive = parse_drive($opt, $conf->{$opt});
4452             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4453                 return 1;
4454             }
4455         }
4456     }
4457
4458     my $scsihwid="scsihw$controller";
4459
4460     vm_deviceunplug($vmid, $conf, $scsihwid);
4461
4462     return 1;
4463 }
4464
4465 sub qemu_add_pci_bridge {
4466     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4467
4468     my $bridges = {};
4469
4470     my $bridgeid;
4471
4472     print_pci_addr($device, $bridges, $arch, $machine_type);
4473
4474     while (my ($k, $v) = each %$bridges) {
4475         $bridgeid = $k;
4476     }
4477     return 1 if !defined($bridgeid) || $bridgeid < 1;
4478
4479     my $bridge = "pci.$bridgeid";
4480     my $devices_list = vm_devices_list($vmid);
4481
4482     if (!defined($devices_list->{$bridge})) {
4483         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4484     }
4485
4486     return 1;
4487 }
4488
4489 sub qemu_set_link_status {
4490     my ($vmid, $device, $up) = @_;
4491
4492     mon_cmd($vmid, "set_link", name => $device,
4493                up => $up ? JSON::true : JSON::false);
4494 }
4495
4496 sub qemu_netdevadd {
4497     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4498
4499     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4500     my %options =  split(/[=,]/, $netdev);
4501
4502     if (defined(my $vhost = $options{vhost})) {
4503         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4504     }
4505
4506     if (defined(my $queues = $options{queues})) {
4507         $options{queues} = $queues + 0;
4508     }
4509
4510     mon_cmd($vmid, "netdev_add",  %options);
4511     return 1;
4512 }
4513
4514 sub qemu_netdevdel {
4515     my ($vmid, $deviceid) = @_;
4516
4517     mon_cmd($vmid, "netdev_del", id => $deviceid);
4518 }
4519
4520 sub qemu_usb_hotplug {
4521     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4522
4523     return if !$device;
4524
4525     # remove the old one first
4526     vm_deviceunplug($vmid, $conf, $deviceid);
4527
4528     # check if xhci controller is necessary and available
4529     my $devicelist = vm_devices_list($vmid);
4530
4531     if (!$devicelist->{xhci}) {
4532         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4533         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4534     }
4535
4536     # add the new one
4537     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4538 }
4539
4540 sub qemu_cpu_hotplug {
4541     my ($vmid, $conf, $vcpus) = @_;
4542
4543     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4544
4545     my $sockets = 1;
4546     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4547     $sockets = $conf->{sockets} if  $conf->{sockets};
4548     my $cores = $conf->{cores} || 1;
4549     my $maxcpus = $sockets * $cores;
4550
4551     $vcpus = $maxcpus if !$vcpus;
4552
4553     die "you can't add more vcpus than maxcpus\n"
4554         if $vcpus > $maxcpus;
4555
4556     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4557
4558     if ($vcpus < $currentvcpus) {
4559
4560         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4561
4562             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4563                 qemu_devicedel($vmid, "cpu$i");
4564                 my $retry = 0;
4565                 my $currentrunningvcpus = undef;
4566                 while (1) {
4567                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4568                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4569                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4570                     $retry++;
4571                     sleep 1;
4572                 }
4573                 #update conf after each succesfull cpu unplug
4574                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4575                 PVE::QemuConfig->write_config($vmid, $conf);
4576             }
4577         } else {
4578             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4579         }
4580
4581         return;
4582     }
4583
4584     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4585     die "vcpus in running vm does not match its configuration\n"
4586         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4587
4588     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4589         my $arch = get_vm_arch($conf);
4590
4591         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4592             my $cpustr = print_cpu_device($conf, $arch, $i);
4593             qemu_deviceadd($vmid, $cpustr);
4594
4595             my $retry = 0;
4596             my $currentrunningvcpus = undef;
4597             while (1) {
4598                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4599                 last if scalar(@{$currentrunningvcpus}) == $i;
4600                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4601                 sleep 1;
4602                 $retry++;
4603             }
4604             #update conf after each succesfull cpu hotplug
4605             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4606             PVE::QemuConfig->write_config($vmid, $conf);
4607         }
4608     } else {
4609
4610         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4611             mon_cmd($vmid, "cpu-add", id => int($i));
4612         }
4613     }
4614 }
4615
4616 sub qemu_block_set_io_throttle {
4617     my ($vmid, $deviceid,
4618         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4619         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4620         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4621         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4622
4623     return if !check_running($vmid) ;
4624
4625     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4626         bps => int($bps),
4627         bps_rd => int($bps_rd),
4628         bps_wr => int($bps_wr),
4629         iops => int($iops),
4630         iops_rd => int($iops_rd),
4631         iops_wr => int($iops_wr),
4632         bps_max => int($bps_max),
4633         bps_rd_max => int($bps_rd_max),
4634         bps_wr_max => int($bps_wr_max),
4635         iops_max => int($iops_max),
4636         iops_rd_max => int($iops_rd_max),
4637         iops_wr_max => int($iops_wr_max),
4638         bps_max_length => int($bps_max_length),
4639         bps_rd_max_length => int($bps_rd_max_length),
4640         bps_wr_max_length => int($bps_wr_max_length),
4641         iops_max_length => int($iops_max_length),
4642         iops_rd_max_length => int($iops_rd_max_length),
4643         iops_wr_max_length => int($iops_wr_max_length),
4644     );
4645
4646 }
4647
4648 sub qemu_block_resize {
4649     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4650
4651     my $running = check_running($vmid);
4652
4653     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4654
4655     return if !$running;
4656
4657     my $padding = (1024 - $size % 1024) % 1024;
4658     $size = $size + $padding;
4659
4660     mon_cmd(
4661         $vmid,
4662         "block_resize",
4663         device => $deviceid,
4664         size => int($size),
4665         timeout => 60,
4666     );
4667 }
4668
4669 sub qemu_volume_snapshot {
4670     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4671
4672     my $running = check_running($vmid);
4673
4674     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4675         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4676     } else {
4677         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4678     }
4679 }
4680
4681 sub qemu_volume_snapshot_delete {
4682     my ($vmid, $storecfg, $volid, $snap) = @_;
4683
4684     my $running = check_running($vmid);
4685     my $attached_deviceid;
4686
4687     if ($running) {
4688         my $conf = PVE::QemuConfig->load_config($vmid);
4689         PVE::QemuConfig->foreach_volume($conf, sub {
4690             my ($ds, $drive) = @_;
4691             $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4692         });
4693     }
4694
4695     if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4696         mon_cmd(
4697             $vmid,
4698             'blockdev-snapshot-delete-internal-sync',
4699             device => $attached_deviceid,
4700             name => $snap,
4701         );
4702     } else {
4703         PVE::Storage::volume_snapshot_delete(
4704             $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4705     }
4706 }
4707
4708 sub set_migration_caps {
4709     my ($vmid, $savevm) = @_;
4710
4711     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4712
4713     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4714     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4715
4716     my $cap_ref = [];
4717
4718     my $enabled_cap = {
4719         "auto-converge" => 1,
4720         "xbzrle" => 1,
4721         "x-rdma-pin-all" => 0,
4722         "zero-blocks" => 0,
4723         "compress" => 0,
4724         "dirty-bitmaps" => $dirty_bitmaps,
4725     };
4726
4727     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4728
4729     for my $supported_capability (@$supported_capabilities) {
4730         push @$cap_ref, {
4731             capability => $supported_capability->{capability},
4732             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4733         };
4734     }
4735
4736     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4737 }
4738
4739 sub foreach_volid {
4740     my ($conf, $func, @param) = @_;
4741
4742     my $volhash = {};
4743
4744     my $test_volid = sub {
4745         my ($key, $drive, $snapname, $pending) = @_;
4746
4747         my $volid = $drive->{file};
4748         return if !$volid;
4749
4750         $volhash->{$volid}->{cdrom} //= 1;
4751         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4752
4753         my $replicate = $drive->{replicate} // 1;
4754         $volhash->{$volid}->{replicate} //= 0;
4755         $volhash->{$volid}->{replicate} = 1 if $replicate;
4756
4757         $volhash->{$volid}->{shared} //= 0;
4758         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4759
4760         $volhash->{$volid}->{is_unused} //= 0;
4761         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4762
4763         $volhash->{$volid}->{is_attached} //= 0;
4764         $volhash->{$volid}->{is_attached} = 1
4765             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4766
4767         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4768             if defined($snapname);
4769
4770         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4771
4772         my $size = $drive->{size};
4773         $volhash->{$volid}->{size} //= $size if $size;
4774
4775         $volhash->{$volid}->{is_vmstate} //= 0;
4776         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4777
4778         $volhash->{$volid}->{is_tpmstate} //= 0;
4779         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4780
4781         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4782     };
4783
4784     my $include_opts = {
4785         extra_keys => ['vmstate'],
4786         include_unused => 1,
4787     };
4788
4789     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4790
4791     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4792         if defined($conf->{pending}) && $conf->{pending}->%*;
4793
4794     foreach my $snapname (keys %{$conf->{snapshots}}) {
4795         my $snap = $conf->{snapshots}->{$snapname};
4796         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4797     }
4798
4799     foreach my $volid (keys %$volhash) {
4800         &$func($volid, $volhash->{$volid}, @param);
4801     }
4802 }
4803
4804 my $fast_plug_option = {
4805     'description' => 1,
4806     'hookscript' => 1,
4807     'lock' => 1,
4808     'migrate_downtime' => 1,
4809     'migrate_speed' => 1,
4810     'name' => 1,
4811     'onboot' => 1,
4812     'protection' => 1,
4813     'shares' => 1,
4814     'startup' => 1,
4815     'tags' => 1,
4816     'vmstatestorage' => 1,
4817 };
4818
4819 for my $opt (keys %$confdesc_cloudinit) {
4820     $fast_plug_option->{$opt} = 1;
4821 };
4822
4823 # hotplug changes in [PENDING]
4824 # $selection hash can be used to only apply specified options, for
4825 # example: { cores => 1 } (only apply changed 'cores')
4826 # $errors ref is used to return error messages
4827 sub vmconfig_hotplug_pending {
4828     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4829
4830     my $defaults = load_defaults();
4831     my $arch = get_vm_arch($conf);
4832     my $machine_type = get_vm_machine($conf, undef, $arch);
4833
4834     # commit values which do not have any impact on running VM first
4835     # Note: those option cannot raise errors, we we do not care about
4836     # $selection and always apply them.
4837
4838     my $add_error = sub {
4839         my ($opt, $msg) = @_;
4840         $errors->{$opt} = "hotplug problem - $msg";
4841     };
4842
4843     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4844
4845     my $cloudinit_record_changed = sub {
4846         my ($conf, $opt, $old, $new) = @_;
4847         return if !$cloudinit_pending_properties->{$opt};
4848
4849         my $ci = ($conf->{cloudinit} //= {});
4850
4851         my $recorded = $ci->{$opt};
4852         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4853
4854         if (defined($new)) {
4855             if (defined($old)) {
4856                 # an existing value is being modified
4857                 if (defined($recorded)) {
4858                     # the value was already not in sync
4859                     if ($new eq $recorded) {
4860                         # a value is being reverted to the cloud-init state:
4861                         delete $ci->{$opt};
4862                         delete $added{$opt};
4863                     } else {
4864                         # the value was changed multiple times, do nothing
4865                     }
4866                 } elsif ($added{$opt}) {
4867                     # the value had been marked as added and is being changed, do nothing
4868                 } else {
4869                     # the value is new, record it:
4870                     $ci->{$opt} = $old;
4871                 }
4872             } else {
4873                 # a new value is being added
4874                 if (defined($recorded)) {
4875                     # it was already not in sync
4876                     if ($new eq $recorded) {
4877                         # a value is being reverted to the cloud-init state:
4878                         delete $ci->{$opt};
4879                         delete $added{$opt};
4880                     } else {
4881                         # the value had temporarily been removed, do nothing
4882                     }
4883                 } elsif ($added{$opt}) {
4884                     # the value had been marked as added already, do nothing
4885                 } else {
4886                     # the value is new, add it
4887                     $added{$opt} = 1;
4888                 }
4889             }
4890         } elsif (!defined($old)) {
4891             # a non-existent value is being removed? ignore...
4892         } else {
4893             # a value is being deleted
4894             if (defined($recorded)) {
4895                 # a value was already recorded, just keep it
4896             } elsif ($added{$opt}) {
4897                 # the value was marked as added, remove it
4898                 delete $added{$opt};
4899             } else {
4900                 # a previously unrecorded value is being removed, record the old value:
4901                 $ci->{$opt} = $old;
4902             }
4903         }
4904
4905         my $added = join(',', sort keys %added);
4906         $ci->{added} = $added if length($added);
4907     };
4908
4909     my $changes = 0;
4910     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4911         if ($fast_plug_option->{$opt}) {
4912             my $new = delete $conf->{pending}->{$opt};
4913             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4914             $conf->{$opt} = $new;
4915             $changes = 1;
4916         }
4917     }
4918
4919     if ($changes) {
4920         PVE::QemuConfig->write_config($vmid, $conf);
4921     }
4922
4923     my $ostype = $conf->{ostype};
4924     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4925     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4926     my $usb_hotplug = $hotplug_features->{usb}
4927         && min_version($version, 7, 1)
4928         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4929
4930     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4931     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4932
4933     foreach my $opt (sort keys %$pending_delete_hash) {
4934         next if $selection && !$selection->{$opt};
4935         my $force = $pending_delete_hash->{$opt}->{force};
4936         eval {
4937             if ($opt eq 'hotplug') {
4938                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4939             } elsif ($opt eq 'tablet') {
4940                 die "skip\n" if !$hotplug_features->{usb};
4941                 if ($defaults->{tablet}) {
4942                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4943                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4944                         if $arch eq 'aarch64';
4945                 } else {
4946                     vm_deviceunplug($vmid, $conf, 'tablet');
4947                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4948                 }
4949             } elsif ($opt =~ m/^usb(\d+)$/) {
4950                 my $index = $1;
4951                 die "skip\n" if !$usb_hotplug;
4952                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4953                 vm_deviceunplug($vmid, $conf, $opt);
4954             } elsif ($opt eq 'vcpus') {
4955                 die "skip\n" if !$hotplug_features->{cpu};
4956                 qemu_cpu_hotplug($vmid, $conf, undef);
4957             } elsif ($opt eq 'balloon') {
4958                 # enable balloon device is not hotpluggable
4959                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4960                 # here we reset the ballooning value to memory
4961                 my $balloon = get_current_memory($conf->{memory});
4962                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4963             } elsif ($fast_plug_option->{$opt}) {
4964                 # do nothing
4965             } elsif ($opt =~ m/^net(\d+)$/) {
4966                 die "skip\n" if !$hotplug_features->{network};
4967                 vm_deviceunplug($vmid, $conf, $opt);
4968                 if($have_sdn) {
4969                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
4970                     PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
4971                 }
4972             } elsif (is_valid_drivename($opt)) {
4973                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4974                 vm_deviceunplug($vmid, $conf, $opt);
4975                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4976             } elsif ($opt =~ m/^memory$/) {
4977                 die "skip\n" if !$hotplug_features->{memory};
4978                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
4979             } elsif ($opt eq 'cpuunits') {
4980                 $cgroup->change_cpu_shares(undef);
4981             } elsif ($opt eq 'cpulimit') {
4982                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4983             } else {
4984                 die "skip\n";
4985             }
4986         };
4987         if (my $err = $@) {
4988             &$add_error($opt, $err) if $err ne "skip\n";
4989         } else {
4990             my $old = delete $conf->{$opt};
4991             $cloudinit_record_changed->($conf, $opt, $old, undef);
4992             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4993         }
4994     }
4995
4996     my $cloudinit_opt;
4997     foreach my $opt (keys %{$conf->{pending}}) {
4998         next if $selection && !$selection->{$opt};
4999         my $value = $conf->{pending}->{$opt};
5000         eval {
5001             if ($opt eq 'hotplug') {
5002                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5003                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
5004             } elsif ($opt eq 'tablet') {
5005                 die "skip\n" if !$hotplug_features->{usb};
5006                 if ($value == 1) {
5007                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5008                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5009                         if $arch eq 'aarch64';
5010                 } elsif ($value == 0) {
5011                     vm_deviceunplug($vmid, $conf, 'tablet');
5012                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5013                 }
5014             } elsif ($opt =~ m/^usb(\d+)$/) {
5015                 my $index = $1;
5016                 die "skip\n" if !$usb_hotplug;
5017                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5018                 my $id = $opt;
5019                 if ($d->{host} =~ m/^spice$/i)  {
5020                     $id = "usbredirdev$index";
5021                 }
5022                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5023             } elsif ($opt eq 'vcpus') {
5024                 die "skip\n" if !$hotplug_features->{cpu};
5025                 qemu_cpu_hotplug($vmid, $conf, $value);
5026             } elsif ($opt eq 'balloon') {
5027                 # enable/disable balloning device is not hotpluggable
5028                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5029                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5030                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5031
5032                 # allow manual ballooning if shares is set to zero
5033                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5034                     my $memory = get_current_memory($conf->{memory});
5035                     my $balloon = $conf->{pending}->{balloon} || $memory;
5036                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5037                 }
5038             } elsif ($opt =~ m/^net(\d+)$/) {
5039                 # some changes can be done without hotplug
5040                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5041                                     $vmid, $opt, $value, $arch, $machine_type);
5042             } elsif (is_valid_drivename($opt)) {
5043                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5044                 # some changes can be done without hotplug
5045                 my $drive = parse_drive($opt, $value);
5046                 if (drive_is_cloudinit($drive)) {
5047                     $cloudinit_opt = [$opt, $drive];
5048                     # apply all the other changes first, then generate the cloudinit disk
5049                     die "skip\n";
5050                 }
5051                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5052                                      $vmid, $opt, $value, $arch, $machine_type);
5053             } elsif ($opt =~ m/^memory$/) { #dimms
5054                 die "skip\n" if !$hotplug_features->{memory};
5055                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5056             } elsif ($opt eq 'cpuunits') {
5057                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5058                 $cgroup->change_cpu_shares($new_cpuunits);
5059             } elsif ($opt eq 'cpulimit') {
5060                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5061                 $cgroup->change_cpu_quota($cpulimit, 100000);
5062             } elsif ($opt eq 'agent') {
5063                 vmconfig_update_agent($conf, $opt, $value);
5064             } else {
5065                 die "skip\n";  # skip non-hot-pluggable options
5066             }
5067         };
5068         if (my $err = $@) {
5069             &$add_error($opt, $err) if $err ne "skip\n";
5070         } else {
5071             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5072             $conf->{$opt} = $value;
5073             delete $conf->{pending}->{$opt};
5074         }
5075     }
5076
5077     if (defined($cloudinit_opt)) {
5078         my ($opt, $drive) = @$cloudinit_opt;
5079         my $value = $conf->{pending}->{$opt};
5080         eval {
5081             my $temp = {%$conf, $opt => $value};
5082             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5083             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5084                                  $vmid, $opt, $value, $arch, $machine_type);
5085         };
5086         if (my $err = $@) {
5087             &$add_error($opt, $err) if $err ne "skip\n";
5088         } else {
5089             $conf->{$opt} = $value;
5090             delete $conf->{pending}->{$opt};
5091         }
5092     }
5093
5094     # unplug xhci controller if no usb device is left
5095     if ($usb_hotplug) {
5096         my $has_usb = 0;
5097         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5098             next if !defined($conf->{"usb$i"});
5099             $has_usb = 1;
5100             last;
5101         }
5102         if (!$has_usb) {
5103             vm_deviceunplug($vmid, $conf, 'xhci');
5104         }
5105     }
5106
5107     PVE::QemuConfig->write_config($vmid, $conf);
5108
5109     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5110         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5111     }
5112 }
5113
5114 sub try_deallocate_drive {
5115     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5116
5117     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5118         my $volid = $drive->{file};
5119         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5120             my $sid = PVE::Storage::parse_volume_id($volid);
5121             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5122
5123             # check if the disk is really unused
5124             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5125                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5126             PVE::Storage::vdisk_free($storecfg, $volid);
5127             return 1;
5128         } else {
5129             # If vm is not owner of this disk remove from config
5130             return 1;
5131         }
5132     }
5133
5134     return;
5135 }
5136
5137 sub vmconfig_delete_or_detach_drive {
5138     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5139
5140     my $drive = parse_drive($opt, $conf->{$opt});
5141
5142     my $rpcenv = PVE::RPCEnvironment::get();
5143     my $authuser = $rpcenv->get_user();
5144
5145     if ($force) {
5146         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5147         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5148     } else {
5149         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5150     }
5151 }
5152
5153
5154
5155 sub vmconfig_apply_pending {
5156     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5157
5158     return if !scalar(keys %{$conf->{pending}});
5159
5160     my $add_apply_error = sub {
5161         my ($opt, $msg) = @_;
5162         my $err_msg = "unable to apply pending change $opt : $msg";
5163         $errors->{$opt} = $err_msg;
5164         warn $err_msg;
5165     };
5166
5167     # cold plug
5168
5169     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5170     foreach my $opt (sort keys %$pending_delete_hash) {
5171         my $force = $pending_delete_hash->{$opt}->{force};
5172         eval {
5173             if ($opt =~ m/^unused/) {
5174                 die "internal error";
5175             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5176                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5177             } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5178                 if($have_sdn) {
5179                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5180                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5181                     warn if $@;
5182                 }
5183             }
5184         };
5185         if (my $err = $@) {
5186             $add_apply_error->($opt, $err);
5187         } else {
5188             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5189             delete $conf->{$opt};
5190         }
5191     }
5192
5193     PVE::QemuConfig->cleanup_pending($conf);
5194
5195     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5196
5197     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5198         next if $opt eq 'delete'; # just to be sure
5199         eval {
5200             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5201                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5202             } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5203                 if($have_sdn) {
5204                     my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5205                     if ($conf->{$opt}){
5206                         my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5207
5208                         if ($old_net->{bridge} ne $new_net->{bridge} ||
5209                             $old_net->{macaddr} ne $new_net->{macaddr}) {
5210                             PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5211                         }
5212                    }
5213                    #fixme: reuse ip if mac change && same bridge
5214                    PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5215                 }
5216             }
5217         };
5218         if (my $err = $@) {
5219             $add_apply_error->($opt, $err);
5220         } else {
5221
5222             if (is_valid_drivename($opt)) {
5223                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5224                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5225             }
5226
5227             $conf->{$opt} = delete $conf->{pending}->{$opt};
5228         }
5229     }
5230
5231     # write all changes at once to avoid unnecessary i/o
5232     PVE::QemuConfig->write_config($vmid, $conf);
5233     if ($generate_cloudinit) {
5234         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5235             # After successful generation and if there were changes to be applied, update the
5236             # config to drop the {cloudinit} entry.
5237             PVE::QemuConfig->write_config($vmid, $conf);
5238         }
5239     }
5240 }
5241
5242 sub vmconfig_update_net {
5243     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5244
5245     my $newnet = parse_net($value);
5246
5247     if ($conf->{$opt}) {
5248         my $oldnet = parse_net($conf->{$opt});
5249
5250         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5251             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5252             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5253             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5254             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5255
5256             # for non online change, we try to hot-unplug
5257             die "skip\n" if !$hotplug;
5258             vm_deviceunplug($vmid, $conf, $opt);
5259
5260             if($have_sdn) {
5261                 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5262             }
5263
5264         } else {
5265
5266             die "internal error" if $opt !~ m/net(\d+)/;
5267             my $iface = "tap${vmid}i$1";
5268
5269             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5270                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5271                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5272                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5273                 PVE::Network::tap_unplug($iface);
5274
5275                 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5276                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5277                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5278                     qemu_set_link_status($vmid, $opt, 0);
5279                 }
5280
5281                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5282                     if ($have_sdn) {
5283                         PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5284                         PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5285                     }
5286                 }
5287
5288                 if ($have_sdn) {
5289                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5290                 } else {
5291                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5292                 }
5293
5294                 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5295                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5296                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5297                     qemu_set_link_status($vmid, $opt, 1);
5298                 }
5299
5300             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5301                 # Rate can be applied on its own but any change above needs to
5302                 # include the rate in tap_plug since OVS resets everything.
5303                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5304             }
5305
5306             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5307                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5308             }
5309
5310             return 1;
5311         }
5312     }
5313
5314     if ($hotplug) {
5315         if ($have_sdn) {
5316             PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5317             PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5318         }
5319         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5320     } else {
5321         die "skip\n";
5322     }
5323 }
5324
5325 sub vmconfig_update_agent {
5326     my ($conf, $opt, $value) = @_;
5327
5328     die "skip\n" if !$conf->{$opt};
5329
5330     my $hotplug_options = { fstrim_cloned_disks => 1 };
5331
5332     my $old_agent = parse_guest_agent($conf);
5333     my $agent = parse_guest_agent({$opt => $value});
5334
5335     for my $option (keys %$agent) { # added/changed options
5336         next if defined($hotplug_options->{$option});
5337         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5338     }
5339
5340     for my $option (keys %$old_agent) { # removed options
5341         next if defined($hotplug_options->{$option});
5342         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5343     }
5344
5345     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5346 }
5347
5348 sub vmconfig_update_disk {
5349     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5350
5351     my $drive = parse_drive($opt, $value);
5352
5353     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5354         my $media = $drive->{media} || 'disk';
5355         my $oldmedia = $old_drive->{media} || 'disk';
5356         die "unable to change media type\n" if $media ne $oldmedia;
5357
5358         if (!drive_is_cdrom($old_drive)) {
5359
5360             if ($drive->{file} ne $old_drive->{file}) {
5361
5362                 die "skip\n" if !$hotplug;
5363
5364                 # unplug and register as unused
5365                 vm_deviceunplug($vmid, $conf, $opt);
5366                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5367
5368             } else {
5369                 # update existing disk
5370
5371                 # skip non hotpluggable value
5372                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5373                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5374                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5375                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5376                     safe_string_ne($drive->{product}, $old_drive->{product}) ||
5377                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5378                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5379                     safe_string_ne($drive->{vendor}, $old_drive->{vendor}) ||
5380                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5381                     die "skip\n";
5382                 }
5383
5384                 # apply throttle
5385                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5386                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5387                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5388                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5389                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5390                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5391                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5392                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5393                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5394                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5395                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5396                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5397                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5398                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5399                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5400                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5401                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5402                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5403
5404                     qemu_block_set_io_throttle(
5405                         $vmid,"drive-$opt",
5406                         ($drive->{mbps} || 0)*1024*1024,
5407                         ($drive->{mbps_rd} || 0)*1024*1024,
5408                         ($drive->{mbps_wr} || 0)*1024*1024,
5409                          $drive->{iops} || 0,
5410                          $drive->{iops_rd} || 0,
5411                          $drive->{iops_wr} || 0,
5412                         ($drive->{mbps_max} || 0)*1024*1024,
5413                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5414                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5415                          $drive->{iops_max} || 0,
5416                          $drive->{iops_rd_max} || 0,
5417                          $drive->{iops_wr_max} || 0,
5418                          $drive->{bps_max_length} || 1,
5419                          $drive->{bps_rd_max_length} || 1,
5420                          $drive->{bps_wr_max_length} || 1,
5421                          $drive->{iops_max_length} || 1,
5422                          $drive->{iops_rd_max_length} || 1,
5423                          $drive->{iops_wr_max_length} || 1,
5424                     );
5425
5426                 }
5427
5428                 return 1;
5429             }
5430
5431         } else { # cdrom
5432
5433             if ($drive->{file} eq 'none') {
5434                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5435                 if (drive_is_cloudinit($old_drive)) {
5436                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5437                 }
5438             } else {
5439                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5440
5441                 # force eject if locked
5442                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5443
5444                 if ($path) {
5445                     mon_cmd($vmid, "blockdev-change-medium",
5446                         id => "$opt", filename => "$path");
5447                 }
5448             }
5449
5450             return 1;
5451         }
5452     }
5453
5454     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5455     # hotplug new disks
5456     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5457     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5458 }
5459
5460 sub vmconfig_update_cloudinit_drive {
5461     my ($storecfg, $conf, $vmid) = @_;
5462
5463     my $cloudinit_ds = undef;
5464     my $cloudinit_drive = undef;
5465
5466     PVE::QemuConfig->foreach_volume($conf, sub {
5467         my ($ds, $drive) = @_;
5468         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5469             $cloudinit_ds = $ds;
5470             $cloudinit_drive = $drive;
5471         }
5472     });
5473
5474     return if !$cloudinit_drive;
5475
5476     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5477         PVE::QemuConfig->write_config($vmid, $conf);
5478     }
5479
5480     my $running = PVE::QemuServer::check_running($vmid);
5481
5482     if ($running) {
5483         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5484         if ($path) {
5485             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5486             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5487         }
5488     }
5489 }
5490
5491 # called in locked context by incoming migration
5492 sub vm_migrate_get_nbd_disks {
5493     my ($storecfg, $conf, $replicated_volumes) = @_;
5494
5495     my $local_volumes = {};
5496     PVE::QemuConfig->foreach_volume($conf, sub {
5497         my ($ds, $drive) = @_;
5498
5499         return if drive_is_cdrom($drive);
5500         return if $ds eq 'tpmstate0';
5501
5502         my $volid = $drive->{file};
5503
5504         return if !$volid;
5505
5506         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5507
5508         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5509         return if $scfg->{shared};
5510
5511         my $format = qemu_img_format($scfg, $volname);
5512
5513         # replicated disks re-use existing state via bitmap
5514         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5515         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5516     });
5517     return $local_volumes;
5518 }
5519
5520 # called in locked context by incoming migration
5521 sub vm_migrate_alloc_nbd_disks {
5522     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5523
5524     my $nbd = {};
5525     foreach my $opt (sort keys %$source_volumes) {
5526         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5527
5528         if ($use_existing) {
5529             $nbd->{$opt}->{drivestr} = print_drive($drive);
5530             $nbd->{$opt}->{volid} = $volid;
5531             $nbd->{$opt}->{replicated} = 1;
5532             next;
5533         }
5534
5535         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5536
5537         # order of precedence, filtered by whether storage supports it:
5538         # 1. explicit requested format
5539         # 2. default format of storage
5540         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5541         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5542
5543         my $size = $drive->{size} / 1024;
5544         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5545         my $newdrive = $drive;
5546         $newdrive->{format} = $format;
5547         $newdrive->{file} = $newvolid;
5548         my $drivestr = print_drive($newdrive);
5549         $nbd->{$opt}->{drivestr} = $drivestr;
5550         $nbd->{$opt}->{volid} = $newvolid;
5551     }
5552
5553     return $nbd;
5554 }
5555
5556 # see vm_start_nolock for parameters, additionally:
5557 # migrate_opts:
5558 #   storagemap = parsed storage map for allocating NBD disks
5559 sub vm_start {
5560     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5561
5562     return PVE::QemuConfig->lock_config($vmid, sub {
5563         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5564
5565         die "you can't start a vm if it's a template\n"
5566             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5567
5568         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5569         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5570
5571         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5572
5573         if ($has_backup_lock && $running) {
5574             # a backup is currently running, attempt to start the guest in the
5575             # existing QEMU instance
5576             return vm_resume($vmid);
5577         }
5578
5579         PVE::QemuConfig->check_lock($conf)
5580             if !($params->{skiplock} || $has_suspended_lock);
5581
5582         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5583
5584         die "VM $vmid already running\n" if $running;
5585
5586         if (my $storagemap = $migrate_opts->{storagemap}) {
5587             my $replicated = $migrate_opts->{replicated_volumes};
5588             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5589             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5590
5591             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5592                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5593             }
5594         }
5595
5596         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5597     });
5598 }
5599
5600
5601 # params:
5602 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5603 #   skiplock => 0/1, skip checking for config lock
5604 #   skiptemplate => 0/1, skip checking whether VM is template
5605 #   forcemachine => to force QEMU machine (rollback/migration)
5606 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5607 #   timeout => in seconds
5608 #   paused => start VM in paused state (backup)
5609 #   resume => resume from hibernation
5610 #   pbs-backing => {
5611 #      sata0 => {
5612 #         repository
5613 #         snapshot
5614 #         keyfile
5615 #         archive
5616 #      },
5617 #      virtio2 => ...
5618 #   }
5619 # migrate_opts:
5620 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5621 #   migratedfrom => source node
5622 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5623 #   network => CIDR of migration network
5624 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5625 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5626 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5627 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5628 #       contained in config
5629 sub vm_start_nolock {
5630     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5631
5632     my $statefile = $params->{statefile};
5633     my $resume = $params->{resume};
5634
5635     my $migratedfrom = $migrate_opts->{migratedfrom};
5636     my $migration_type = $migrate_opts->{type};
5637
5638     my $res = {};
5639
5640     # clean up leftover reboot request files
5641     eval { clear_reboot_request($vmid); };
5642     warn $@ if $@;
5643
5644     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5645         vmconfig_apply_pending($vmid, $conf, $storecfg);
5646         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5647     }
5648
5649     # don't regenerate the ISO if the VM is started as part of a live migration
5650     # this way we can reuse the old ISO with the correct config
5651     if (!$migratedfrom) {
5652         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5653             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5654             # $conf->{cloudinit}, so we could just not do this?
5655             # But we do it above, so for now let's be consistent.
5656             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5657         }
5658     }
5659
5660     # override offline migrated volumes, conf is out of date still
5661     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5662         for my $key (sort keys $offline_volumes->%*) {
5663             my $parsed = parse_drive($key, $conf->{$key});
5664             $parsed->{file} = $offline_volumes->{$key};
5665             $conf->{$key} = print_drive($parsed);
5666         }
5667     }
5668
5669     my $defaults = load_defaults();
5670
5671     # set environment variable useful inside network script
5672     # for remote migration the config is available on the target node!
5673     if (!$migrate_opts->{remote_node}) {
5674         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5675     }
5676
5677     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5678
5679     my $forcemachine = $params->{forcemachine};
5680     my $forcecpu = $params->{forcecpu};
5681     if ($resume) {
5682         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5683         $forcemachine = $conf->{runningmachine};
5684         $forcecpu = $conf->{runningcpu};
5685         print "Resuming suspended VM\n";
5686     }
5687
5688     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5689         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5690
5691     my $migration_ip;
5692     my $get_migration_ip = sub {
5693         my ($nodename) = @_;
5694
5695         return $migration_ip if defined($migration_ip);
5696
5697         my $cidr = $migrate_opts->{network};
5698
5699         if (!defined($cidr)) {
5700             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5701             $cidr = $dc_conf->{migration}->{network};
5702         }
5703
5704         if (defined($cidr)) {
5705             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5706
5707             die "could not get IP: no address configured on local " .
5708                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5709
5710             die "could not get IP: multiple addresses configured on local " .
5711                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5712
5713             $migration_ip = @$ips[0];
5714         }
5715
5716         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5717             if !defined($migration_ip);
5718
5719         return $migration_ip;
5720     };
5721
5722     if ($statefile) {
5723         if ($statefile eq 'tcp') {
5724             my $migrate = $res->{migrate} = { proto => 'tcp' };
5725             $migrate->{addr} = "localhost";
5726             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5727             my $nodename = nodename();
5728
5729             if (!defined($migration_type)) {
5730                 if (defined($datacenterconf->{migration}->{type})) {
5731                     $migration_type = $datacenterconf->{migration}->{type};
5732                 } else {
5733                     $migration_type = 'secure';
5734                 }
5735             }
5736
5737             if ($migration_type eq 'insecure') {
5738                 $migrate->{addr} = $get_migration_ip->($nodename);
5739                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5740             }
5741
5742             # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5743             # via QMP later
5744             push @$cmd, '-incoming', 'defer';
5745             push @$cmd, '-S';
5746
5747         } elsif ($statefile eq 'unix') {
5748             # should be default for secure migrations as a ssh TCP forward
5749             # tunnel is not deterministic reliable ready and fails regurarly
5750             # to set up in time, so use UNIX socket forwards
5751             my $migrate = $res->{migrate} = { proto => 'unix' };
5752             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5753             unlink $migrate->{addr};
5754
5755             $migrate->{uri} = "unix:$migrate->{addr}";
5756             push @$cmd, '-incoming', $migrate->{uri};
5757             push @$cmd, '-S';
5758
5759         } elsif (-e $statefile) {
5760             push @$cmd, '-loadstate', $statefile;
5761         } else {
5762             my $statepath = PVE::Storage::path($storecfg, $statefile);
5763             push @$vollist, $statefile;
5764             push @$cmd, '-loadstate', $statepath;
5765         }
5766     } elsif ($params->{paused}) {
5767         push @$cmd, '-S';
5768     }
5769
5770     my $memory = get_current_memory($conf->{memory});
5771     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5772
5773     my $pci_reserve_list = [];
5774     for my $device (values $pci_devices->%*) {
5775         next if $device->{mdev}; # we don't reserve for mdev devices
5776         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5777     }
5778
5779     # reserve all PCI IDs before actually doing anything with them
5780     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5781
5782     eval {
5783         my $uuid;
5784         for my $id (sort keys %$pci_devices) {
5785             my $d = $pci_devices->{$id};
5786             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5787
5788             my $chosen_mdev;
5789             for my $dev ($d->{ids}->@*) {
5790                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5791                 if ($d->{mdev}) {
5792                     warn $@ if $@;
5793                     $chosen_mdev = $info;
5794                     last if $chosen_mdev; # if successful, we're done
5795                 } else {
5796                     die $@ if $@;
5797                 }
5798             }
5799
5800             next if !$d->{mdev};
5801             die "could not create mediated device\n" if !defined($chosen_mdev);
5802
5803             # nvidia grid needs the uuid of the mdev as qemu parameter
5804             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5805                 if (defined($conf->{smbios1})) {
5806                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5807                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5808                 }
5809                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5810             }
5811         }
5812         push @$cmd, '-uuid', $uuid if defined($uuid);
5813     };
5814     if (my $err = $@) {
5815         eval { cleanup_pci_devices($vmid, $conf) };
5816         warn $@ if $@;
5817         die $err;
5818     }
5819
5820     PVE::Storage::activate_volumes($storecfg, $vollist);
5821
5822
5823     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5824     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5825     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5826     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5827     # timeout should be more than enough here...
5828     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5829
5830     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5831
5832     my %run_params = (
5833         timeout => $statefile ? undef : $start_timeout,
5834         umask => 0077,
5835         noerr => 1,
5836     );
5837
5838     # when migrating, prefix QEMU output so other side can pick up any
5839     # errors that might occur and show the user
5840     if ($migratedfrom) {
5841         $run_params{quiet} = 1;
5842         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5843     }
5844
5845     my %systemd_properties = (
5846         Slice => 'qemu.slice',
5847         KillMode => 'process',
5848         SendSIGKILL => 0,
5849         TimeoutStopUSec => ULONG_MAX, # infinity
5850     );
5851
5852     if (PVE::CGroup::cgroup_mode() == 2) {
5853         $systemd_properties{CPUWeight} = $cpuunits;
5854     } else {
5855         $systemd_properties{CPUShares} = $cpuunits;
5856     }
5857
5858     if (my $cpulimit = $conf->{cpulimit}) {
5859         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5860     }
5861     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5862
5863     my $run_qemu = sub {
5864         PVE::Tools::run_fork sub {
5865             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5866
5867             my $tpmpid;
5868             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5869                 # start the TPM emulator so QEMU can connect on start
5870                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5871             }
5872
5873             my $exitcode = run_command($cmd, %run_params);
5874             if ($exitcode) {
5875                 if ($tpmpid) {
5876                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5877                     kill 'TERM', $tpmpid;
5878                 }
5879                 die "QEMU exited with code $exitcode\n";
5880             }
5881         };
5882     };
5883
5884     if ($conf->{hugepages}) {
5885
5886         my $code = sub {
5887             my $hotplug_features =
5888                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5889             my $hugepages_topology =
5890                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5891
5892             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5893
5894             PVE::QemuServer::Memory::hugepages_mount();
5895             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5896
5897             eval { $run_qemu->() };
5898             if (my $err = $@) {
5899                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5900                     if !$conf->{keephugepages};
5901                 die $err;
5902             }
5903
5904             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5905                 if !$conf->{keephugepages};
5906         };
5907         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5908
5909     } else {
5910         eval { $run_qemu->() };
5911     }
5912
5913     if (my $err = $@) {
5914         # deactivate volumes if start fails
5915         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5916         warn $@ if $@;
5917         eval { cleanup_pci_devices($vmid, $conf) };
5918         warn $@ if $@;
5919
5920         die "start failed: $err";
5921     }
5922
5923     # re-reserve all PCI IDs now that we can know the actual VM PID
5924     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5925     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5926     warn $@ if $@;
5927
5928     if (defined(my $migrate = $res->{migrate})) {
5929         if ($migrate->{proto} eq 'tcp') {
5930             my $nodename = nodename();
5931             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5932             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5933             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5934             mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
5935         }
5936         print "migration listens on $migrate->{uri}\n";
5937     } elsif ($statefile) {
5938         eval { mon_cmd($vmid, "cont"); };
5939         warn $@ if $@;
5940     }
5941
5942     #start nbd server for storage migration
5943     if (my $nbd = $migrate_opts->{nbd}) {
5944         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5945
5946         my $migrate_storage_uri;
5947         # nbd_protocol_version > 0 for unix socket support
5948         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5949             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5950             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5951             $migrate_storage_uri = "nbd:unix:$socket_path";
5952             $res->{migrate}->{unix_sockets} = [$socket_path];
5953         } else {
5954             my $nodename = nodename();
5955             my $localip = $get_migration_ip->($nodename);
5956             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5957             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5958
5959             mon_cmd($vmid, "nbd-server-start", addr => {
5960                 type => 'inet',
5961                 data => {
5962                     host => "${localip}",
5963                     port => "${storage_migrate_port}",
5964                 },
5965             });
5966             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5967             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5968         }
5969
5970         my $block_info = mon_cmd($vmid, "query-block");
5971         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5972
5973         foreach my $opt (sort keys %$nbd) {
5974             my $drivestr = $nbd->{$opt}->{drivestr};
5975             my $volid = $nbd->{$opt}->{volid};
5976
5977             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5978
5979             mon_cmd(
5980                 $vmid,
5981                 "block-export-add",
5982                 id => "drive-$opt",
5983                 'node-name' => $block_node,
5984                 writable => JSON::true,
5985                 type => "nbd",
5986                 name => "drive-$opt", # NBD export name
5987             );
5988
5989             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5990             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5991             print "re-using replicated volume: $opt - $volid\n"
5992                 if $nbd->{$opt}->{replicated};
5993
5994             $res->{drives}->{$opt} = $nbd->{$opt};
5995             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5996         }
5997     }
5998
5999     if ($migratedfrom) {
6000         eval {
6001             set_migration_caps($vmid);
6002         };
6003         warn $@ if $@;
6004
6005         if ($spice_port) {
6006             print "spice listens on port $spice_port\n";
6007             $res->{spice_port} = $spice_port;
6008             if ($migrate_opts->{spice_ticket}) {
6009                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6010                     $migrate_opts->{spice_ticket});
6011                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6012             }
6013         }
6014
6015     } else {
6016         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6017             if !$statefile && $conf->{balloon};
6018
6019         foreach my $opt (keys %$conf) {
6020             next if $opt !~  m/^net\d+$/;
6021             my $nicconf = parse_net($conf->{$opt});
6022             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6023         }
6024         add_nets_bridge_fdb($conf, $vmid);
6025     }
6026
6027    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6028         eval {
6029             mon_cmd(
6030                 $vmid,
6031                 'qom-set',
6032                 path => "machine/peripheral/balloon0",
6033                 property => "guest-stats-polling-interval",
6034                 value => 2
6035             );
6036         };
6037         log_warn("could not set polling interval for ballooning - $@") if $@;
6038     }
6039
6040     if ($resume) {
6041         print "Resumed VM, removing state\n";
6042         if (my $vmstate = $conf->{vmstate}) {
6043             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6044             PVE::Storage::vdisk_free($storecfg, $vmstate);
6045         }
6046         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6047         PVE::QemuConfig->write_config($vmid, $conf);
6048     }
6049
6050     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6051
6052     my ($current_machine, $is_deprecated) =
6053         PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6054     if ($is_deprecated) {
6055         log_warn(
6056             "current machine version '$current_machine' is deprecated - see the documentation and ".
6057             "change to a newer one",
6058         );
6059     }
6060
6061     return $res;
6062 }
6063
6064 sub vm_commandline {
6065     my ($storecfg, $vmid, $snapname) = @_;
6066
6067     my $conf = PVE::QemuConfig->load_config($vmid);
6068
6069     my ($forcemachine, $forcecpu);
6070     if ($snapname) {
6071         my $snapshot = $conf->{snapshots}->{$snapname};
6072         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6073
6074         # check for machine or CPU overrides in snapshot
6075         $forcemachine = $snapshot->{runningmachine};
6076         $forcecpu = $snapshot->{runningcpu};
6077
6078         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6079
6080         $conf = $snapshot;
6081     }
6082
6083     my $defaults = load_defaults();
6084
6085     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6086
6087     return PVE::Tools::cmd2string($cmd);
6088 }
6089
6090 sub vm_reset {
6091     my ($vmid, $skiplock) = @_;
6092
6093     PVE::QemuConfig->lock_config($vmid, sub {
6094
6095         my $conf = PVE::QemuConfig->load_config($vmid);
6096
6097         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6098
6099         mon_cmd($vmid, "system_reset");
6100     });
6101 }
6102
6103 sub get_vm_volumes {
6104     my ($conf) = @_;
6105
6106     my $vollist = [];
6107     foreach_volid($conf, sub {
6108         my ($volid, $attr) = @_;
6109
6110         return if $volid =~ m|^/|;
6111
6112         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6113         return if !$sid;
6114
6115         push @$vollist, $volid;
6116     });
6117
6118     return $vollist;
6119 }
6120
6121 sub cleanup_pci_devices {
6122     my ($vmid, $conf) = @_;
6123
6124     foreach my $key (keys %$conf) {
6125         next if $key !~ m/^hostpci(\d+)$/;
6126         my $hostpciindex = $1;
6127         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6128         my $d = parse_hostpci($conf->{$key});
6129         if ($d->{mdev}) {
6130             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6131             # don't want to break ABI just for this two liner
6132             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6133
6134             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6135             # out when we do it first. so wait for up to 10 seconds and then try it manually
6136             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/ && -e $dev_sysfs_dir) {
6137                 my $count = 0;
6138                 while (-e $dev_sysfs_dir && $count < 10) {
6139                     sleep 1;
6140                     $count++;
6141                 }
6142                 print "waited $count seconds for mediated device driver finishing clean up\n";
6143             }
6144
6145             if (-e $dev_sysfs_dir) {
6146                 print "actively clean up mediated device with UUID $uuid\n";
6147                 PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1");
6148             }
6149         }
6150     }
6151     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6152 }
6153
6154 sub vm_stop_cleanup {
6155     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6156
6157     eval {
6158
6159         if (!$keepActive) {
6160             my $vollist = get_vm_volumes($conf);
6161             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6162
6163             if (my $tpmdrive = $conf->{tpmstate0}) {
6164                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6165                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6166                 if ($storeid) {
6167                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6168                 }
6169             }
6170         }
6171
6172         foreach my $ext (qw(mon qmp pid vnc qga)) {
6173             unlink "/var/run/qemu-server/${vmid}.$ext";
6174         }
6175
6176         if ($conf->{ivshmem}) {
6177             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6178             # just delete it for now, VMs which have this already open do not
6179             # are affected, but new VMs will get a separated one. If this
6180             # becomes an issue we either add some sort of ref-counting or just
6181             # add a "don't delete on stop" flag to the ivshmem format.
6182             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6183         }
6184
6185         cleanup_pci_devices($vmid, $conf);
6186
6187         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6188     };
6189     warn $@ if $@; # avoid errors - just warn
6190 }
6191
6192 # call only in locked context
6193 sub _do_vm_stop {
6194     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6195
6196     my $pid = check_running($vmid, $nocheck);
6197     return if !$pid;
6198
6199     my $conf;
6200     if (!$nocheck) {
6201         $conf = PVE::QemuConfig->load_config($vmid);
6202         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6203         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6204             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6205             $timeout = $opts->{down} if $opts->{down};
6206         }
6207         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6208     }
6209
6210     eval {
6211         if ($shutdown) {
6212             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6213                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6214             } else {
6215                 mon_cmd($vmid, "system_powerdown");
6216             }
6217         } else {
6218             mon_cmd($vmid, "quit");
6219         }
6220     };
6221     my $err = $@;
6222
6223     if (!$err) {
6224         $timeout = 60 if !defined($timeout);
6225
6226         my $count = 0;
6227         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6228             $count++;
6229             sleep 1;
6230         }
6231
6232         if ($count >= $timeout) {
6233             if ($force) {
6234                 warn "VM still running - terminating now with SIGTERM\n";
6235                 kill 15, $pid;
6236             } else {
6237                 die "VM quit/powerdown failed - got timeout\n";
6238             }
6239         } else {
6240             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6241             return;
6242         }
6243     } else {
6244        if (!check_running($vmid, $nocheck)) {
6245             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6246             return;
6247        }
6248        if ($force) {
6249             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6250             kill 15, $pid;
6251         } else {
6252             die "VM quit/powerdown failed\n";
6253         }
6254     }
6255
6256     # wait again
6257     $timeout = 10;
6258
6259     my $count = 0;
6260     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6261         $count++;
6262         sleep 1;
6263     }
6264
6265     if ($count >= $timeout) {
6266         warn "VM still running - terminating now with SIGKILL\n";
6267         kill 9, $pid;
6268         sleep 1;
6269     }
6270
6271     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6272 }
6273
6274 # Note: use $nocheck to skip tests if VM configuration file exists.
6275 # We need that when migration VMs to other nodes (files already moved)
6276 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6277 sub vm_stop {
6278     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6279
6280     $force = 1 if !defined($force) && !$shutdown;
6281
6282     if ($migratedfrom){
6283         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6284         kill 15, $pid if $pid;
6285         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6286         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6287         return;
6288     }
6289
6290     PVE::QemuConfig->lock_config($vmid, sub {
6291         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6292    });
6293 }
6294
6295 sub vm_reboot {
6296     my ($vmid, $timeout) = @_;
6297
6298     PVE::QemuConfig->lock_config($vmid, sub {
6299         eval {
6300
6301             # only reboot if running, as qmeventd starts it again on a stop event
6302             return if !check_running($vmid);
6303
6304             create_reboot_request($vmid);
6305
6306             my $storecfg = PVE::Storage::config();
6307             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6308
6309         };
6310         if (my $err = $@) {
6311             # avoid that the next normal shutdown will be confused for a reboot
6312             clear_reboot_request($vmid);
6313             die $err;
6314         }
6315    });
6316 }
6317
6318 # note: if using the statestorage parameter, the caller has to check privileges
6319 sub vm_suspend {
6320     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6321
6322     my $conf;
6323     my $path;
6324     my $storecfg;
6325     my $vmstate;
6326
6327     PVE::QemuConfig->lock_config($vmid, sub {
6328
6329         $conf = PVE::QemuConfig->load_config($vmid);
6330
6331         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6332         PVE::QemuConfig->check_lock($conf)
6333             if !($skiplock || $is_backing_up);
6334
6335         die "cannot suspend to disk during backup\n"
6336             if $is_backing_up && $includestate;
6337
6338         if ($includestate) {
6339             $conf->{lock} = 'suspending';
6340             my $date = strftime("%Y-%m-%d", localtime(time()));
6341             $storecfg = PVE::Storage::config();
6342             if (!$statestorage) {
6343                 $statestorage = find_vmstate_storage($conf, $storecfg);
6344                 # check permissions for the storage
6345                 my $rpcenv = PVE::RPCEnvironment::get();
6346                 if ($rpcenv->{type} ne 'cli') {
6347                     my $authuser = $rpcenv->get_user();
6348                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6349                 }
6350             }
6351
6352
6353             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6354                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6355             $path = PVE::Storage::path($storecfg, $vmstate);
6356             PVE::QemuConfig->write_config($vmid, $conf);
6357         } else {
6358             mon_cmd($vmid, "stop");
6359         }
6360     });
6361
6362     if ($includestate) {
6363         # save vm state
6364         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6365
6366         eval {
6367             set_migration_caps($vmid, 1);
6368             mon_cmd($vmid, "savevm-start", statefile => $path);
6369             for(;;) {
6370                 my $state = mon_cmd($vmid, "query-savevm");
6371                 if (!$state->{status}) {
6372                     die "savevm not active\n";
6373                 } elsif ($state->{status} eq 'active') {
6374                     sleep(1);
6375                     next;
6376                 } elsif ($state->{status} eq 'completed') {
6377                     print "State saved, quitting\n";
6378                     last;
6379                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6380                     die "query-savevm failed with error '$state->{error}'\n"
6381                 } else {
6382                     die "query-savevm returned status '$state->{status}'\n";
6383                 }
6384             }
6385         };
6386         my $err = $@;
6387
6388         PVE::QemuConfig->lock_config($vmid, sub {
6389             $conf = PVE::QemuConfig->load_config($vmid);
6390             if ($err) {
6391                 # cleanup, but leave suspending lock, to indicate something went wrong
6392                 eval {
6393                     mon_cmd($vmid, "savevm-end");
6394                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6395                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6396                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6397                     PVE::QemuConfig->write_config($vmid, $conf);
6398                 };
6399                 warn $@ if $@;
6400                 die $err;
6401             }
6402
6403             die "lock changed unexpectedly\n"
6404                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6405
6406             mon_cmd($vmid, "quit");
6407             $conf->{lock} = 'suspended';
6408             PVE::QemuConfig->write_config($vmid, $conf);
6409         });
6410     }
6411 }
6412
6413 # $nocheck is set when called as part of a migration - in this context the
6414 # location of the config file (source or target node) is not deterministic,
6415 # since migration cannot wait for pmxcfs to process the rename
6416 sub vm_resume {
6417     my ($vmid, $skiplock, $nocheck) = @_;
6418
6419     PVE::QemuConfig->lock_config($vmid, sub {
6420         my $res = mon_cmd($vmid, 'query-status');
6421         my $resume_cmd = 'cont';
6422         my $reset = 0;
6423         my $conf;
6424         if ($nocheck) {
6425             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6426             if ($@) {
6427                 my $vmlist = PVE::Cluster::get_vmlist();
6428                 if (exists($vmlist->{ids}->{$vmid})) {
6429                     my $node = $vmlist->{ids}->{$vmid}->{node};
6430                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6431                 }
6432                 if (!$conf) {
6433                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6434                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6435                 }
6436             }
6437         } else {
6438             $conf = PVE::QemuConfig->load_config($vmid);
6439         }
6440
6441         if ($res->{status}) {
6442             return if $res->{status} eq 'running'; # job done, go home
6443             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6444             $reset = 1 if $res->{status} eq 'shutdown';
6445         }
6446
6447         if (!$nocheck) {
6448             PVE::QemuConfig->check_lock($conf)
6449                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6450         }
6451
6452         if ($reset) {
6453             # required if a VM shuts down during a backup and we get a resume
6454             # request before the backup finishes for example
6455             mon_cmd($vmid, "system_reset");
6456         }
6457
6458         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6459
6460         mon_cmd($vmid, $resume_cmd);
6461     });
6462 }
6463
6464 sub vm_sendkey {
6465     my ($vmid, $skiplock, $key) = @_;
6466
6467     PVE::QemuConfig->lock_config($vmid, sub {
6468
6469         my $conf = PVE::QemuConfig->load_config($vmid);
6470
6471         # there is no qmp command, so we use the human monitor command
6472         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6473         die $res if $res ne '';
6474     });
6475 }
6476
6477 sub check_bridge_access {
6478     my ($rpcenv, $authuser, $conf) = @_;
6479
6480     return 1 if $authuser eq 'root@pam';
6481
6482     for my $opt (sort keys $conf->%*) {
6483         next if $opt !~ m/^net\d+$/;
6484         my $net = parse_net($conf->{$opt});
6485         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6486         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6487     }
6488     return 1;
6489 };
6490
6491 sub check_mapping_access {
6492     my ($rpcenv, $user, $conf) = @_;
6493
6494     for my $opt (keys $conf->%*) {
6495         if ($opt =~ m/^usb\d+$/) {
6496             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6497             if (my $host = $device->{host}) {
6498                 die "only root can set '$opt' config for real devices\n"
6499                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6500             } elsif ($device->{mapping}) {
6501                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6502             } else {
6503                 die "either 'host' or 'mapping' must be set.\n";
6504             }
6505         } elsif ($opt =~ m/^hostpci\d+$/) {
6506             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6507             if ($device->{host}) {
6508                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6509             } elsif ($device->{mapping}) {
6510                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6511             } else {
6512                 die "either 'host' or 'mapping' must be set.\n";
6513             }
6514        }
6515    }
6516 };
6517
6518 sub check_restore_permissions {
6519     my ($rpcenv, $user, $conf) = @_;
6520
6521     check_bridge_access($rpcenv, $user, $conf);
6522     check_mapping_access($rpcenv, $user, $conf);
6523 }
6524 # vzdump restore implementaion
6525
6526 sub tar_archive_read_firstfile {
6527     my $archive = shift;
6528
6529     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6530
6531     # try to detect archive type first
6532     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6533         die "unable to open file '$archive'\n";
6534     my $firstfile = <$fh>;
6535     kill 15, $pid;
6536     close $fh;
6537
6538     die "ERROR: archive contaions no data\n" if !$firstfile;
6539     chomp $firstfile;
6540
6541     return $firstfile;
6542 }
6543
6544 sub tar_restore_cleanup {
6545     my ($storecfg, $statfile) = @_;
6546
6547     print STDERR "starting cleanup\n";
6548
6549     if (my $fd = IO::File->new($statfile, "r")) {
6550         while (defined(my $line = <$fd>)) {
6551             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6552                 my $volid = $2;
6553                 eval {
6554                     if ($volid =~ m|^/|) {
6555                         unlink $volid || die 'unlink failed\n';
6556                     } else {
6557                         PVE::Storage::vdisk_free($storecfg, $volid);
6558                     }
6559                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6560                 };
6561                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6562             } else {
6563                 print STDERR "unable to parse line in statfile - $line";
6564             }
6565         }
6566         $fd->close();
6567     }
6568 }
6569
6570 sub restore_file_archive {
6571     my ($archive, $vmid, $user, $opts) = @_;
6572
6573     return restore_vma_archive($archive, $vmid, $user, $opts)
6574         if $archive eq '-';
6575
6576     my $info = PVE::Storage::archive_info($archive);
6577     my $format = $opts->{format} // $info->{format};
6578     my $comp = $info->{compression};
6579
6580     # try to detect archive format
6581     if ($format eq 'tar') {
6582         return restore_tar_archive($archive, $vmid, $user, $opts);
6583     } else {
6584         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6585     }
6586 }
6587
6588 # hepler to remove disks that will not be used after restore
6589 my $restore_cleanup_oldconf = sub {
6590     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6591
6592     my $kept_disks = {};
6593
6594     PVE::QemuConfig->foreach_volume($oldconf, sub {
6595         my ($ds, $drive) = @_;
6596
6597         return if drive_is_cdrom($drive, 1);
6598
6599         my $volid = $drive->{file};
6600         return if !$volid || $volid =~ m|^/|;
6601
6602         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6603         return if !$path || !$owner || ($owner != $vmid);
6604
6605         # Note: only delete disk we want to restore
6606         # other volumes will become unused
6607         if ($virtdev_hash->{$ds}) {
6608             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6609             if (my $err = $@) {
6610                 warn $err;
6611             }
6612         } else {
6613             $kept_disks->{$volid} = 1;
6614         }
6615     });
6616
6617     # after the restore we have no snapshots anymore
6618     for my $snapname (keys $oldconf->{snapshots}->%*) {
6619         my $snap = $oldconf->{snapshots}->{$snapname};
6620         if ($snap->{vmstate}) {
6621             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6622             if (my $err = $@) {
6623                 warn $err;
6624             }
6625         }
6626
6627         for my $volid (keys $kept_disks->%*) {
6628             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6629             warn $@ if $@;
6630         }
6631     }
6632 };
6633
6634 # Helper to parse vzdump backup device hints
6635 #
6636 # $rpcenv: Environment, used to ckeck storage permissions
6637 # $user: User ID, to check storage permissions
6638 # $storecfg: Storage configuration
6639 # $fh: the file handle for reading the configuration
6640 # $devinfo: should contain device sizes for all backu-up'ed devices
6641 # $options: backup options (pool, default storage)
6642 #
6643 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6644 my $parse_backup_hints = sub {
6645     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6646
6647     my $check_storage = sub { # assert if an image can be allocate
6648         my ($storeid, $scfg) = @_;
6649         die "Content type 'images' is not available on storage '$storeid'\n"
6650             if !$scfg->{content}->{images};
6651         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6652             if $user ne 'root@pam';
6653     };
6654
6655     my $virtdev_hash = {};
6656     while (defined(my $line = <$fh>)) {
6657         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6658             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6659             die "archive does not contain data for drive '$virtdev'\n"
6660                 if !$devinfo->{$devname};
6661
6662             if (defined($options->{storage})) {
6663                 $storeid = $options->{storage} || 'local';
6664             } elsif (!$storeid) {
6665                 $storeid = 'local';
6666             }
6667             $format = 'raw' if !$format;
6668             $devinfo->{$devname}->{devname} = $devname;
6669             $devinfo->{$devname}->{virtdev} = $virtdev;
6670             $devinfo->{$devname}->{format} = $format;
6671             $devinfo->{$devname}->{storeid} = $storeid;
6672
6673             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6674             $check_storage->($storeid, $scfg); # permission and content type check
6675
6676             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6677         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6678             my $virtdev = $1;
6679             my $drive = parse_drive($virtdev, $2);
6680
6681             if (drive_is_cloudinit($drive)) {
6682                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6683                 $storeid = $options->{storage} if defined ($options->{storage});
6684                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6685                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6686
6687                 $check_storage->($storeid, $scfg); # permission and content type check
6688
6689                 $virtdev_hash->{$virtdev} = {
6690                     format => $format,
6691                     storeid => $storeid,
6692                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6693                     is_cloudinit => 1,
6694                 };
6695             }
6696         }
6697     }
6698
6699     return $virtdev_hash;
6700 };
6701
6702 # Helper to allocate and activate all volumes required for a restore
6703 #
6704 # $storecfg: Storage configuration
6705 # $virtdev_hash: as returned by parse_backup_hints()
6706 #
6707 # Returns: { $virtdev => $volid }
6708 my $restore_allocate_devices = sub {
6709     my ($storecfg, $virtdev_hash, $vmid) = @_;
6710
6711     my $map = {};
6712     foreach my $virtdev (sort keys %$virtdev_hash) {
6713         my $d = $virtdev_hash->{$virtdev};
6714         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6715         my $storeid = $d->{storeid};
6716         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6717
6718         # test if requested format is supported
6719         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6720         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6721         $d->{format} = $defFormat if !$supported;
6722
6723         my $name;
6724         if ($d->{is_cloudinit}) {
6725             $name = "vm-$vmid-cloudinit";
6726             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6727             if ($scfg->{path}) {
6728                 $name .= ".$d->{format}";
6729             }
6730         }
6731
6732         my $volid = PVE::Storage::vdisk_alloc(
6733             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6734
6735         print STDERR "new volume ID is '$volid'\n";
6736         $d->{volid} = $volid;
6737
6738         PVE::Storage::activate_volumes($storecfg, [$volid]);
6739
6740         $map->{$virtdev} = $volid;
6741     }
6742
6743     return $map;
6744 };
6745
6746 sub restore_update_config_line {
6747     my ($cookie, $map, $line, $unique) = @_;
6748
6749     return '' if $line =~ m/^\#qmdump\#/;
6750     return '' if $line =~ m/^\#vzdump\#/;
6751     return '' if $line =~ m/^lock:/;
6752     return '' if $line =~ m/^unused\d+:/;
6753     return '' if $line =~ m/^parent:/;
6754
6755     my $res = '';
6756
6757     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6758     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6759         # try to convert old 1.X settings
6760         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6761         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6762             my ($model, $macaddr) = split(/\=/, $devconfig);
6763             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6764             my $net = {
6765                 model => $model,
6766                 bridge => "vmbr$ind",
6767                 macaddr => $macaddr,
6768             };
6769             my $netstr = print_net($net);
6770
6771             $res .= "net$cookie->{netcount}: $netstr\n";
6772             $cookie->{netcount}++;
6773         }
6774     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6775         my ($id, $netstr) = ($1, $2);
6776         my $net = parse_net($netstr);
6777         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6778         $netstr = print_net($net);
6779         $res .= "$id: $netstr\n";
6780     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6781         my $virtdev = $1;
6782         my $value = $3;
6783         my $di = parse_drive($virtdev, $value);
6784         if (defined($di->{backup}) && !$di->{backup}) {
6785             $res .= "#$line";
6786         } elsif ($map->{$virtdev}) {
6787             delete $di->{format}; # format can change on restore
6788             $di->{file} = $map->{$virtdev};
6789             $value = print_drive($di);
6790             $res .= "$virtdev: $value\n";
6791         } else {
6792             $res .= $line;
6793         }
6794     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6795         my $vmgenid = $1;
6796         if ($vmgenid ne '0') {
6797             # always generate a new vmgenid if there was a valid one setup
6798             $vmgenid = generate_uuid();
6799         }
6800         $res .= "vmgenid: $vmgenid\n";
6801     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6802         my ($uuid, $uuid_str);
6803         UUID::generate($uuid);
6804         UUID::unparse($uuid, $uuid_str);
6805         my $smbios1 = parse_smbios1($2);
6806         $smbios1->{uuid} = $uuid_str;
6807         $res .= $1.print_smbios1($smbios1)."\n";
6808     } else {
6809         $res .= $line;
6810     }
6811
6812     return $res;
6813 }
6814
6815 my $restore_deactivate_volumes = sub {
6816     my ($storecfg, $virtdev_hash) = @_;
6817
6818     my $vollist = [];
6819     for my $dev (values $virtdev_hash->%*) {
6820         push $vollist->@*, $dev->{volid} if $dev->{volid};
6821     }
6822
6823     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6824     print STDERR $@ if $@;
6825 };
6826
6827 my $restore_destroy_volumes = sub {
6828     my ($storecfg, $virtdev_hash) = @_;
6829
6830     for my $dev (values $virtdev_hash->%*) {
6831         my $volid = $dev->{volid} or next;
6832         eval {
6833             PVE::Storage::vdisk_free($storecfg, $volid);
6834             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6835         };
6836         print STDERR "unable to cleanup '$volid' - $@" if $@;
6837     }
6838 };
6839
6840 sub restore_merge_config {
6841     my ($filename, $backup_conf_raw, $override_conf) = @_;
6842
6843     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6844     for my $key (keys $override_conf->%*) {
6845         $backup_conf->{$key} = $override_conf->{$key};
6846     }
6847
6848     return $backup_conf;
6849 }
6850
6851 sub scan_volids {
6852     my ($cfg, $vmid) = @_;
6853
6854     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6855
6856     my $volid_hash = {};
6857     foreach my $storeid (keys %$info) {
6858         foreach my $item (@{$info->{$storeid}}) {
6859             next if !($item->{volid} && $item->{size});
6860             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6861             $volid_hash->{$item->{volid}} = $item;
6862         }
6863     }
6864
6865     return $volid_hash;
6866 }
6867
6868 sub update_disk_config {
6869     my ($vmid, $conf, $volid_hash) = @_;
6870
6871     my $changes;
6872     my $prefix = "VM $vmid";
6873
6874     # used and unused disks
6875     my $referenced = {};
6876
6877     # Note: it is allowed to define multiple storages with same path (alias), so
6878     # we need to check both 'volid' and real 'path' (two different volid can point
6879     # to the same path).
6880
6881     my $referencedpath = {};
6882
6883     # update size info
6884     PVE::QemuConfig->foreach_volume($conf, sub {
6885         my ($opt, $drive) = @_;
6886
6887         my $volid = $drive->{file};
6888         return if !$volid;
6889         my $volume = $volid_hash->{$volid};
6890
6891         # mark volid as "in-use" for next step
6892         $referenced->{$volid} = 1;
6893         if ($volume && (my $path = $volume->{path})) {
6894             $referencedpath->{$path} = 1;
6895         }
6896
6897         return if drive_is_cdrom($drive);
6898         return if !$volume;
6899
6900         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6901         if (defined($updated)) {
6902             $changes = 1;
6903             $conf->{$opt} = print_drive($updated);
6904             print "$prefix ($opt): $msg\n";
6905         }
6906     });
6907
6908     # remove 'unusedX' entry if volume is used
6909     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6910         my ($opt, $drive) = @_;
6911
6912         my $volid = $drive->{file};
6913         return if !$volid;
6914
6915         my $path;
6916         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6917         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6918             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6919             $changes = 1;
6920             delete $conf->{$opt};
6921         }
6922
6923         $referenced->{$volid} = 1;
6924         $referencedpath->{$path} = 1 if $path;
6925     });
6926
6927     foreach my $volid (sort keys %$volid_hash) {
6928         next if $volid =~ m/vm-$vmid-state-/;
6929         next if $referenced->{$volid};
6930         my $path = $volid_hash->{$volid}->{path};
6931         next if !$path; # just to be sure
6932         next if $referencedpath->{$path};
6933         $changes = 1;
6934         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6935         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6936         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6937     }
6938
6939     return $changes;
6940 }
6941
6942 sub rescan {
6943     my ($vmid, $nolock, $dryrun) = @_;
6944
6945     my $cfg = PVE::Storage::config();
6946
6947     print "rescan volumes...\n";
6948     my $volid_hash = scan_volids($cfg, $vmid);
6949
6950     my $updatefn =  sub {
6951         my ($vmid) = @_;
6952
6953         my $conf = PVE::QemuConfig->load_config($vmid);
6954
6955         PVE::QemuConfig->check_lock($conf);
6956
6957         my $vm_volids = {};
6958         foreach my $volid (keys %$volid_hash) {
6959             my $info = $volid_hash->{$volid};
6960             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6961         }
6962
6963         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6964
6965         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6966     };
6967
6968     if (defined($vmid)) {
6969         if ($nolock) {
6970             &$updatefn($vmid);
6971         } else {
6972             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6973         }
6974     } else {
6975         my $vmlist = config_list();
6976         foreach my $vmid (keys %$vmlist) {
6977             if ($nolock) {
6978                 &$updatefn($vmid);
6979             } else {
6980                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6981             }
6982         }
6983     }
6984 }
6985
6986 sub restore_proxmox_backup_archive {
6987     my ($archive, $vmid, $user, $options) = @_;
6988
6989     my $storecfg = PVE::Storage::config();
6990
6991     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6992     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6993
6994     my $fingerprint = $scfg->{fingerprint};
6995     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6996
6997     my $repo = PVE::PBSClient::get_repository($scfg);
6998     my $namespace = $scfg->{namespace};
6999
7000     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7001     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7002     local $ENV{PBS_PASSWORD} = $password;
7003     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7004
7005     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7006         PVE::Storage::parse_volname($storecfg, $archive);
7007
7008     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7009
7010     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7011
7012     my $tmpdir = "/var/tmp/vzdumptmp$$";
7013     rmtree $tmpdir;
7014     mkpath $tmpdir;
7015
7016     my $conffile = PVE::QemuConfig->config_file($vmid);
7017      # disable interrupts (always do cleanups)
7018     local $SIG{INT} =
7019         local $SIG{TERM} =
7020         local $SIG{QUIT} =
7021         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7022
7023     # Note: $oldconf is undef if VM does not exists
7024     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7025     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7026     my $new_conf_raw = '';
7027
7028     my $rpcenv = PVE::RPCEnvironment::get();
7029     my $devinfo = {}; # info about drives included in backup
7030     my $virtdev_hash = {}; # info about allocated drives
7031
7032     eval {
7033         # enable interrupts
7034         local $SIG{INT} =
7035             local $SIG{TERM} =
7036             local $SIG{QUIT} =
7037             local $SIG{HUP} =
7038             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7039
7040         my $cfgfn = "$tmpdir/qemu-server.conf";
7041         my $firewall_config_fn = "$tmpdir/fw.conf";
7042         my $index_fn = "$tmpdir/index.json";
7043
7044         my $cmd = "restore";
7045
7046         my $param = [$pbs_backup_name, "index.json", $index_fn];
7047         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7048         my $index = PVE::Tools::file_get_contents($index_fn);
7049         $index = decode_json($index);
7050
7051         foreach my $info (@{$index->{files}}) {
7052             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7053                 my $devname = $1;
7054                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7055                     $devinfo->{$devname}->{size} = $1;
7056                 } else {
7057                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7058                 }
7059             }
7060         }
7061
7062         my $is_qemu_server_backup = scalar(
7063             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7064         );
7065         if (!$is_qemu_server_backup) {
7066             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7067         }
7068         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7069
7070         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7071         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7072
7073         if ($has_firewall_config) {
7074             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7075             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7076
7077             my $pve_firewall_dir = '/etc/pve/firewall';
7078             mkdir $pve_firewall_dir; # make sure the dir exists
7079             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7080         }
7081
7082         my $fh = IO::File->new($cfgfn, "r") ||
7083             die "unable to read qemu-server.conf - $!\n";
7084
7085         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7086
7087         # fixme: rate limit?
7088
7089         # create empty/temp config
7090         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7091
7092         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7093
7094         # allocate volumes
7095         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7096
7097         foreach my $virtdev (sort keys %$virtdev_hash) {
7098             my $d = $virtdev_hash->{$virtdev};
7099             next if $d->{is_cloudinit}; # no need to restore cloudinit
7100
7101             # this fails if storage is unavailable
7102             my $volid = $d->{volid};
7103             my $path = PVE::Storage::path($storecfg, $volid);
7104
7105             # for live-restore we only want to preload the efidisk and TPM state
7106             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7107
7108             my @ns_arg;
7109             if (defined(my $ns = $scfg->{namespace})) {
7110                 @ns_arg = ('--ns', $ns);
7111             }
7112
7113             my $pbs_restore_cmd = [
7114                 '/usr/bin/pbs-restore',
7115                 '--repository', $repo,
7116                 @ns_arg,
7117                 $pbs_backup_name,
7118                 "$d->{devname}.img.fidx",
7119                 $path,
7120                 '--verbose',
7121                 ];
7122
7123             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7124             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7125
7126             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7127                 push @$pbs_restore_cmd, '--skip-zero';
7128             }
7129
7130             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7131             print "restore proxmox backup image: $dbg_cmdstring\n";
7132             run_command($pbs_restore_cmd);
7133         }
7134
7135         $fh->seek(0, 0) || die "seek failed - $!\n";
7136
7137         my $cookie = { netcount => 0 };
7138         while (defined(my $line = <$fh>)) {
7139             $new_conf_raw .= restore_update_config_line(
7140                 $cookie,
7141                 $map,
7142                 $line,
7143                 $options->{unique},
7144             );
7145         }
7146
7147         $fh->close();
7148     };
7149     my $err = $@;
7150
7151     if ($err || !$options->{live}) {
7152         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7153     }
7154
7155     rmtree $tmpdir;
7156
7157     if ($err) {
7158         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7159         die $err;
7160     }
7161
7162     if ($options->{live}) {
7163         # keep lock during live-restore
7164         $new_conf_raw .= "\nlock: create";
7165     }
7166
7167     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7168     check_restore_permissions($rpcenv, $user, $new_conf);
7169     PVE::QemuConfig->write_config($vmid, $new_conf);
7170
7171     eval { rescan($vmid, 1); };
7172     warn $@ if $@;
7173
7174     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7175
7176     if ($options->{live}) {
7177         # enable interrupts
7178         local $SIG{INT} =
7179             local $SIG{TERM} =
7180             local $SIG{QUIT} =
7181             local $SIG{HUP} =
7182             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7183
7184         my $conf = PVE::QemuConfig->load_config($vmid);
7185         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7186
7187         # these special drives are already restored before start
7188         delete $devinfo->{'drive-efidisk0'};
7189         delete $devinfo->{'drive-tpmstate0-backup'};
7190
7191         my $pbs_opts = {
7192             repo => $repo,
7193             keyfile => $keyfile,
7194             snapshot => $pbs_backup_name,
7195             namespace => $namespace,
7196         };
7197         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7198
7199         PVE::QemuConfig->remove_lock($vmid, "create");
7200     }
7201 }
7202
7203 sub pbs_live_restore {
7204     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7205
7206     print "starting VM for live-restore\n";
7207     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7208
7209     my $pbs_backing = {};
7210     for my $ds (keys %$restored_disks) {
7211         $ds =~ m/^drive-(.*)$/;
7212         my $confname = $1;
7213         $pbs_backing->{$confname} = {
7214             repository => $opts->{repo},
7215             snapshot => $opts->{snapshot},
7216             archive => "$ds.img.fidx",
7217         };
7218         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7219         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7220
7221         my $drive = parse_drive($confname, $conf->{$confname});
7222         print "restoring '$ds' to '$drive->{file}'\n";
7223     }
7224
7225     my $drives_streamed = 0;
7226     eval {
7227         # make sure HA doesn't interrupt our restore by stopping the VM
7228         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7229             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7230         }
7231
7232         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7233         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7234         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7235
7236         my $qmeventd_fd = register_qmeventd_handle($vmid);
7237
7238         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7239         # this will effectively collapse the backing image chain consisting of
7240         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7241         # removes itself once all backing images vanish with 'auto-remove=on')
7242         my $jobs = {};
7243         for my $ds (sort keys %$restored_disks) {
7244             my $job_id = "restore-$ds";
7245             mon_cmd($vmid, 'block-stream',
7246                 'job-id' => $job_id,
7247                 device => "$ds",
7248             );
7249             $jobs->{$job_id} = {};
7250         }
7251
7252         mon_cmd($vmid, 'cont');
7253         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7254
7255         print "restore-drive jobs finished successfully, removing all tracking block devices"
7256             ." to disconnect from Proxmox Backup Server\n";
7257
7258         for my $ds (sort keys %$restored_disks) {
7259             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7260         }
7261
7262         close($qmeventd_fd);
7263     };
7264
7265     my $err = $@;
7266
7267     if ($err) {
7268         warn "An error occurred during live-restore: $err\n";
7269         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7270         die "live-restore failed\n";
7271     }
7272 }
7273
7274 sub restore_vma_archive {
7275     my ($archive, $vmid, $user, $opts, $comp) = @_;
7276
7277     my $readfrom = $archive;
7278
7279     my $cfg = PVE::Storage::config();
7280     my $commands = [];
7281     my $bwlimit = $opts->{bwlimit};
7282
7283     my $dbg_cmdstring = '';
7284     my $add_pipe = sub {
7285         my ($cmd) = @_;
7286         push @$commands, $cmd;
7287         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7288         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7289         $readfrom = '-';
7290     };
7291
7292     my $input = undef;
7293     if ($archive eq '-') {
7294         $input = '<&STDIN';
7295     } else {
7296         # If we use a backup from a PVE defined storage we also consider that
7297         # storage's rate limit:
7298         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7299         if (defined($volid)) {
7300             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7301             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7302             if ($readlimit) {
7303                 print STDERR "applying read rate limit: $readlimit\n";
7304                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7305                 $add_pipe->($cstream);
7306             }
7307         }
7308     }
7309
7310     if ($comp) {
7311         my $info = PVE::Storage::decompressor_info('vma', $comp);
7312         my $cmd = $info->{decompressor};
7313         push @$cmd, $readfrom;
7314         $add_pipe->($cmd);
7315     }
7316
7317     my $tmpdir = "/var/tmp/vzdumptmp$$";
7318     rmtree $tmpdir;
7319
7320     # disable interrupts (always do cleanups)
7321     local $SIG{INT} =
7322         local $SIG{TERM} =
7323         local $SIG{QUIT} =
7324         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7325
7326     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7327     POSIX::mkfifo($mapfifo, 0600);
7328     my $fifofh;
7329     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7330
7331     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7332
7333     my $devinfo = {}; # info about drives included in backup
7334     my $virtdev_hash = {}; # info about allocated drives
7335
7336     my $rpcenv = PVE::RPCEnvironment::get();
7337
7338     my $conffile = PVE::QemuConfig->config_file($vmid);
7339
7340     # Note: $oldconf is undef if VM does not exist
7341     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7342     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7343     my $new_conf_raw = '';
7344
7345     my %storage_limits;
7346
7347     my $print_devmap = sub {
7348         my $cfgfn = "$tmpdir/qemu-server.conf";
7349
7350         # we can read the config - that is already extracted
7351         my $fh = IO::File->new($cfgfn, "r") ||
7352             die "unable to read qemu-server.conf - $!\n";
7353
7354         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7355         if (-f $fwcfgfn) {
7356             my $pve_firewall_dir = '/etc/pve/firewall';
7357             mkdir $pve_firewall_dir; # make sure the dir exists
7358             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7359         }
7360
7361         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7362
7363         foreach my $info (values %{$virtdev_hash}) {
7364             my $storeid = $info->{storeid};
7365             next if defined($storage_limits{$storeid});
7366
7367             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7368             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7369             $storage_limits{$storeid} = $limit * 1024;
7370         }
7371
7372         foreach my $devname (keys %$devinfo) {
7373             die "found no device mapping information for device '$devname'\n"
7374                 if !$devinfo->{$devname}->{virtdev};
7375         }
7376
7377         # create empty/temp config
7378         if ($oldconf) {
7379             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7380             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7381         }
7382
7383         # allocate volumes
7384         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7385
7386         # print restore information to $fifofh
7387         foreach my $virtdev (sort keys %$virtdev_hash) {
7388             my $d = $virtdev_hash->{$virtdev};
7389             next if $d->{is_cloudinit}; # no need to restore cloudinit
7390
7391             my $storeid = $d->{storeid};
7392             my $volid = $d->{volid};
7393
7394             my $map_opts = '';
7395             if (my $limit = $storage_limits{$storeid}) {
7396                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7397             }
7398
7399             my $write_zeros = 1;
7400             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7401                 $write_zeros = 0;
7402             }
7403
7404             my $path = PVE::Storage::path($cfg, $volid);
7405
7406             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7407
7408             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7409         }
7410
7411         $fh->seek(0, 0) || die "seek failed - $!\n";
7412
7413         my $cookie = { netcount => 0 };
7414         while (defined(my $line = <$fh>)) {
7415             $new_conf_raw .= restore_update_config_line(
7416                 $cookie,
7417                 $map,
7418                 $line,
7419                 $opts->{unique},
7420             );
7421         }
7422
7423         $fh->close();
7424     };
7425
7426     my $oldtimeout;
7427
7428     eval {
7429         # enable interrupts
7430         local $SIG{INT} =
7431             local $SIG{TERM} =
7432             local $SIG{QUIT} =
7433             local $SIG{HUP} =
7434             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7435         local $SIG{ALRM} = sub { die "got timeout\n"; };
7436
7437         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7438
7439         my $parser = sub {
7440             my $line = shift;
7441
7442             print "$line\n";
7443
7444             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7445                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7446                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7447             } elsif ($line =~ m/^CTIME: /) {
7448                 # we correctly received the vma config, so we can disable
7449                 # the timeout now for disk allocation
7450                 alarm($oldtimeout || 0);
7451                 $oldtimeout = undef;
7452                 &$print_devmap();
7453                 print $fifofh "done\n";
7454                 close($fifofh);
7455                 $fifofh = undef;
7456             }
7457         };
7458
7459         print "restore vma archive: $dbg_cmdstring\n";
7460         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7461     };
7462     my $err = $@;
7463
7464     alarm($oldtimeout) if $oldtimeout;
7465
7466     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7467
7468     close($fifofh) if $fifofh;
7469     unlink $mapfifo;
7470     rmtree $tmpdir;
7471
7472     if ($err) {
7473         $restore_destroy_volumes->($cfg, $virtdev_hash);
7474         die $err;
7475     }
7476
7477     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7478     check_restore_permissions($rpcenv, $user, $new_conf);
7479     PVE::QemuConfig->write_config($vmid, $new_conf);
7480
7481     eval { rescan($vmid, 1); };
7482     warn $@ if $@;
7483
7484     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7485 }
7486
7487 sub restore_tar_archive {
7488     my ($archive, $vmid, $user, $opts) = @_;
7489
7490     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7491         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7492         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7493     }
7494
7495     if ($archive ne '-') {
7496         my $firstfile = tar_archive_read_firstfile($archive);
7497         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7498             if $firstfile ne 'qemu-server.conf';
7499     }
7500
7501     my $storecfg = PVE::Storage::config();
7502
7503     # avoid zombie disks when restoring over an existing VM -> cleanup first
7504     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7505     # skiplock=1 because qmrestore has set the 'create' lock itself already
7506     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7507     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7508
7509     my $tocmd = "/usr/lib/qemu-server/qmextract";
7510
7511     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7512     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7513     $tocmd .= ' --prealloc' if $opts->{prealloc};
7514     $tocmd .= ' --info' if $opts->{info};
7515
7516     # tar option "xf" does not autodetect compression when read from STDIN,
7517     # so we pipe to zcat
7518     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7519         PVE::Tools::shellquote("--to-command=$tocmd");
7520
7521     my $tmpdir = "/var/tmp/vzdumptmp$$";
7522     mkpath $tmpdir;
7523
7524     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7525     local $ENV{VZDUMP_VMID} = $vmid;
7526     local $ENV{VZDUMP_USER} = $user;
7527
7528     my $conffile = PVE::QemuConfig->config_file($vmid);
7529     my $new_conf_raw = '';
7530
7531     # disable interrupts (always do cleanups)
7532     local $SIG{INT} =
7533         local $SIG{TERM} =
7534         local $SIG{QUIT} =
7535         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7536
7537     eval {
7538         # enable interrupts
7539         local $SIG{INT} =
7540             local $SIG{TERM} =
7541             local $SIG{QUIT} =
7542             local $SIG{HUP} =
7543             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7544
7545         if ($archive eq '-') {
7546             print "extracting archive from STDIN\n";
7547             run_command($cmd, input => "<&STDIN");
7548         } else {
7549             print "extracting archive '$archive'\n";
7550             run_command($cmd);
7551         }
7552
7553         return if $opts->{info};
7554
7555         # read new mapping
7556         my $map = {};
7557         my $statfile = "$tmpdir/qmrestore.stat";
7558         if (my $fd = IO::File->new($statfile, "r")) {
7559             while (defined (my $line = <$fd>)) {
7560                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7561                     $map->{$1} = $2 if $1;
7562                 } else {
7563                     print STDERR "unable to parse line in statfile - $line\n";
7564                 }
7565             }
7566             $fd->close();
7567         }
7568
7569         my $confsrc = "$tmpdir/qemu-server.conf";
7570
7571         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7572
7573         my $cookie = { netcount => 0 };
7574         while (defined (my $line = <$srcfd>)) {
7575             $new_conf_raw .= restore_update_config_line(
7576                 $cookie,
7577                 $map,
7578                 $line,
7579                 $opts->{unique},
7580             );
7581         }
7582
7583         $srcfd->close();
7584     };
7585     if (my $err = $@) {
7586         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7587         die $err;
7588     }
7589
7590     rmtree $tmpdir;
7591
7592     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7593
7594     PVE::Cluster::cfs_update(); # make sure we read new file
7595
7596     eval { rescan($vmid, 1); };
7597     warn $@ if $@;
7598 };
7599
7600 sub foreach_storage_used_by_vm {
7601     my ($conf, $func) = @_;
7602
7603     my $sidhash = {};
7604
7605     PVE::QemuConfig->foreach_volume($conf, sub {
7606         my ($ds, $drive) = @_;
7607         return if drive_is_cdrom($drive);
7608
7609         my $volid = $drive->{file};
7610
7611         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7612         $sidhash->{$sid} = $sid if $sid;
7613     });
7614
7615     foreach my $sid (sort keys %$sidhash) {
7616         &$func($sid);
7617     }
7618 }
7619
7620 my $qemu_snap_storage = {
7621     rbd => 1,
7622 };
7623 sub do_snapshots_with_qemu {
7624     my ($storecfg, $volid, $deviceid) = @_;
7625
7626     return if $deviceid =~ m/tpmstate0/;
7627
7628     my $storage_name = PVE::Storage::parse_volume_id($volid);
7629     my $scfg = $storecfg->{ids}->{$storage_name};
7630     die "could not find storage '$storage_name'\n" if !defined($scfg);
7631
7632     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7633         return 1;
7634     }
7635
7636     if ($volid =~ m/\.(qcow2|qed)$/){
7637         return 1;
7638     }
7639
7640     return;
7641 }
7642
7643 sub qga_check_running {
7644     my ($vmid, $nowarn) = @_;
7645
7646     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7647     if ($@) {
7648         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7649         return 0;
7650     }
7651     return 1;
7652 }
7653
7654 sub template_create {
7655     my ($vmid, $conf, $disk) = @_;
7656
7657     my $storecfg = PVE::Storage::config();
7658
7659     PVE::QemuConfig->foreach_volume($conf, sub {
7660         my ($ds, $drive) = @_;
7661
7662         return if drive_is_cdrom($drive);
7663         return if $disk && $ds ne $disk;
7664
7665         my $volid = $drive->{file};
7666         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7667
7668         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7669         $drive->{file} = $voliddst;
7670         $conf->{$ds} = print_drive($drive);
7671         PVE::QemuConfig->write_config($vmid, $conf);
7672     });
7673 }
7674
7675 sub convert_iscsi_path {
7676     my ($path) = @_;
7677
7678     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7679         my $portal = $1;
7680         my $target = $2;
7681         my $lun = $3;
7682
7683         my $initiator_name = get_initiator_name();
7684
7685         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7686                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7687     }
7688
7689     die "cannot convert iscsi path '$path', unkown format\n";
7690 }
7691
7692 sub qemu_img_convert {
7693     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7694
7695     my $storecfg = PVE::Storage::config();
7696     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7697     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7698
7699     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7700
7701     my $cachemode;
7702     my $src_path;
7703     my $src_is_iscsi = 0;
7704     my $src_format;
7705
7706     if ($src_storeid) {
7707         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7708         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7709         $src_format = qemu_img_format($src_scfg, $src_volname);
7710         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7711         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7712         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7713     } elsif (-f $src_volid || -b $src_volid) {
7714         $src_path = $src_volid;
7715         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7716             $src_format = $1;
7717         }
7718     }
7719
7720     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7721
7722     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7723     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7724     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7725     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7726
7727     my $cmd = [];
7728     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7729     push @$cmd, '-l', "snapshot.name=$snapname"
7730         if $snapname && $src_format && $src_format eq "qcow2";
7731     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7732     push @$cmd, '-T', $cachemode if defined($cachemode);
7733     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7734
7735     if ($src_is_iscsi) {
7736         push @$cmd, '--image-opts';
7737         $src_path = convert_iscsi_path($src_path);
7738     } elsif ($src_format) {
7739         push @$cmd, '-f', $src_format;
7740     }
7741
7742     if ($dst_is_iscsi) {
7743         push @$cmd, '--target-image-opts';
7744         $dst_path = convert_iscsi_path($dst_path);
7745     } else {
7746         push @$cmd, '-O', $dst_format;
7747     }
7748
7749     push @$cmd, $src_path;
7750
7751     if (!$dst_is_iscsi && $is_zero_initialized) {
7752         push @$cmd, "zeroinit:$dst_path";
7753     } else {
7754         push @$cmd, $dst_path;
7755     }
7756
7757     my $parser = sub {
7758         my $line = shift;
7759         if($line =~ m/\((\S+)\/100\%\)/){
7760             my $percent = $1;
7761             my $transferred = int($size * $percent / 100);
7762             my $total_h = render_bytes($size, 1);
7763             my $transferred_h = render_bytes($transferred, 1);
7764
7765             print "transferred $transferred_h of $total_h ($percent%)\n";
7766         }
7767
7768     };
7769
7770     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7771     my $err = $@;
7772     die "copy failed: $err" if $err;
7773 }
7774
7775 sub qemu_img_format {
7776     my ($scfg, $volname) = @_;
7777
7778     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7779         return $1;
7780     } else {
7781         return "raw";
7782     }
7783 }
7784
7785 sub qemu_drive_mirror {
7786     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7787
7788     $jobs = {} if !$jobs;
7789
7790     my $qemu_target;
7791     my $format;
7792     $jobs->{"drive-$drive"} = {};
7793
7794     if ($dst_volid =~ /^nbd:/) {
7795         $qemu_target = $dst_volid;
7796         $format = "nbd";
7797     } else {
7798         my $storecfg = PVE::Storage::config();
7799         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7800
7801         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7802
7803         $format = qemu_img_format($dst_scfg, $dst_volname);
7804
7805         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7806
7807         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7808     }
7809
7810     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7811     $opts->{format} = $format if $format;
7812
7813     if (defined($src_bitmap)) {
7814         $opts->{sync} = 'incremental';
7815         $opts->{bitmap} = $src_bitmap;
7816         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7817     }
7818
7819     if (defined($bwlimit)) {
7820         $opts->{speed} = $bwlimit * 1024;
7821         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7822     } else {
7823         print "drive mirror is starting for drive-$drive\n";
7824     }
7825
7826     # if a job already runs for this device we get an error, catch it for cleanup
7827     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7828     if (my $err = $@) {
7829         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7830         warn "$@\n" if $@;
7831         die "mirroring error: $err\n";
7832     }
7833
7834     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7835 }
7836
7837 # $completion can be either
7838 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7839 # 'cancel': wait until all jobs are ready, block-job-cancel them
7840 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7841 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7842 sub qemu_drive_mirror_monitor {
7843     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7844
7845     $completion //= 'complete';
7846     $op //= "mirror";
7847
7848     eval {
7849         my $err_complete = 0;
7850
7851         my $starttime = time ();
7852         while (1) {
7853             die "block job ('$op') timed out\n" if $err_complete > 300;
7854
7855             my $stats = mon_cmd($vmid, "query-block-jobs");
7856             my $ctime = time();
7857
7858             my $running_jobs = {};
7859             for my $stat (@$stats) {
7860                 next if $stat->{type} ne $op;
7861                 $running_jobs->{$stat->{device}} = $stat;
7862             }
7863
7864             my $readycounter = 0;
7865
7866             for my $job_id (sort keys %$jobs) {
7867                 my $job = $running_jobs->{$job_id};
7868
7869                 my $vanished = !defined($job);
7870                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7871                 if($complete || ($vanished && $completion eq 'auto')) {
7872                     print "$job_id: $op-job finished\n";
7873                     delete $jobs->{$job_id};
7874                     next;
7875                 }
7876
7877                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7878
7879                 my $busy = $job->{busy};
7880                 my $ready = $job->{ready};
7881                 if (my $total = $job->{len}) {
7882                     my $transferred = $job->{offset} || 0;
7883                     my $remaining = $total - $transferred;
7884                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7885
7886                     my $duration = $ctime - $starttime;
7887                     my $total_h = render_bytes($total, 1);
7888                     my $transferred_h = render_bytes($transferred, 1);
7889
7890                     my $status = sprintf(
7891                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7892                         render_duration($duration),
7893                     );
7894
7895                     if ($ready) {
7896                         if ($busy) {
7897                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7898                         } else {
7899                             $status .= ", ready";
7900                         }
7901                     }
7902                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7903                     $jobs->{$job_id}->{ready} = $ready;
7904                 }
7905
7906                 $readycounter++ if $job->{ready};
7907             }
7908
7909             last if scalar(keys %$jobs) == 0;
7910
7911             if ($readycounter == scalar(keys %$jobs)) {
7912                 print "all '$op' jobs are ready\n";
7913
7914                 # do the complete later (or has already been done)
7915                 last if $completion eq 'skip' || $completion eq 'auto';
7916
7917                 if ($vmiddst && $vmiddst != $vmid) {
7918                     my $agent_running = $qga && qga_check_running($vmid);
7919                     if ($agent_running) {
7920                         print "freeze filesystem\n";
7921                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7922                         warn $@ if $@;
7923                     } else {
7924                         print "suspend vm\n";
7925                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7926                         warn $@ if $@;
7927                     }
7928
7929                     # if we clone a disk for a new target vm, we don't switch the disk
7930                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7931
7932                     if ($agent_running) {
7933                         print "unfreeze filesystem\n";
7934                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7935                         warn $@ if $@;
7936                     } else {
7937                         print "resume vm\n";
7938                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7939                         warn $@ if $@;
7940                     }
7941
7942                     last;
7943                 } else {
7944
7945                     for my $job_id (sort keys %$jobs) {
7946                         # try to switch the disk if source and destination are on the same guest
7947                         print "$job_id: Completing block job_id...\n";
7948
7949                         my $op;
7950                         if ($completion eq 'complete') {
7951                             $op = 'block-job-complete';
7952                         } elsif ($completion eq 'cancel') {
7953                             $op = 'block-job-cancel';
7954                         } else {
7955                             die "invalid completion value: $completion\n";
7956                         }
7957                         eval { mon_cmd($vmid, $op, device => $job_id) };
7958                         if ($@ =~ m/cannot be completed/) {
7959                             print "$job_id: block job cannot be completed, trying again.\n";
7960                             $err_complete++;
7961                         }else {
7962                             print "$job_id: Completed successfully.\n";
7963                             $jobs->{$job_id}->{complete} = 1;
7964                         }
7965                     }
7966                 }
7967             }
7968             sleep 1;
7969         }
7970     };
7971     my $err = $@;
7972
7973     if ($err) {
7974         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7975         die "block job ($op) error: $err";
7976     }
7977 }
7978
7979 sub qemu_blockjobs_cancel {
7980     my ($vmid, $jobs) = @_;
7981
7982     foreach my $job (keys %$jobs) {
7983         print "$job: Cancelling block job\n";
7984         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7985         $jobs->{$job}->{cancel} = 1;
7986     }
7987
7988     while (1) {
7989         my $stats = mon_cmd($vmid, "query-block-jobs");
7990
7991         my $running_jobs = {};
7992         foreach my $stat (@$stats) {
7993             $running_jobs->{$stat->{device}} = $stat;
7994         }
7995
7996         foreach my $job (keys %$jobs) {
7997
7998             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7999                 print "$job: Done.\n";
8000                 delete $jobs->{$job};
8001             }
8002         }
8003
8004         last if scalar(keys %$jobs) == 0;
8005
8006         sleep 1;
8007     }
8008 }
8009
8010 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8011 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8012 my sub clone_disk_check_io_uring {
8013     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8014
8015     return if !$use_drive_mirror;
8016
8017     # Don't complain when not changing storage.
8018     # Assume if it works for the source, it'll work for the target too.
8019     return if $src_storeid eq $dst_storeid;
8020
8021     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8022     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8023
8024     my $cache_direct = drive_uses_cache_direct($src_drive);
8025
8026     my $src_uses_io_uring;
8027     if ($src_drive->{aio}) {
8028         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8029     } else {
8030         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8031     }
8032
8033     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8034         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8035 }
8036
8037 sub clone_disk {
8038     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8039
8040     my ($vmid, $running) = $source->@{qw(vmid running)};
8041     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8042
8043     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8044     my ($storage, $format) = $dest->@{qw(storage format)};
8045
8046     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8047
8048     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8049         die "cloning from/to EFI disk requires EFI disk\n"
8050             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8051         die "cloning from/to TPM state requires TPM state\n"
8052             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8053
8054         # This would lead to two device nodes in QEMU pointing to the same backing image!
8055         die "cannot change drive name when cloning disk from/to the same VM\n"
8056             if $use_drive_mirror && $vmid == $newvmid;
8057     }
8058
8059     die "cannot move TPM state while VM is running\n"
8060         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8061
8062     my $newvolid;
8063
8064     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8065     print "$src_drivename " if $src_drivename;
8066     print "($drive->{file})\n";
8067
8068     if (!$full) {
8069         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8070         push @$newvollist, $newvolid;
8071     } else {
8072         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8073         my $storeid = $storage || $src_storeid;
8074
8075         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8076
8077         my $name = undef;
8078         my $size = undef;
8079         if (drive_is_cloudinit($drive)) {
8080             $name = "vm-$newvmid-cloudinit";
8081             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8082             if ($scfg->{path}) {
8083                 $name .= ".$dst_format";
8084             }
8085             $snapname = undef;
8086             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8087         } elsif ($dst_drivename eq 'efidisk0') {
8088             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8089         } elsif ($dst_drivename eq 'tpmstate0') {
8090             $dst_format = 'raw';
8091             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8092         } else {
8093             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8094
8095             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8096         }
8097         $newvolid = PVE::Storage::vdisk_alloc(
8098             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8099         );
8100         push @$newvollist, $newvolid;
8101
8102         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8103
8104         if (drive_is_cloudinit($drive)) {
8105             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8106             # if this is the case, we have to complete any block-jobs still there from
8107             # previous drive-mirrors
8108             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8109                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8110             }
8111             goto no_data_clone;
8112         }
8113
8114         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8115         if ($use_drive_mirror) {
8116             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8117                 $completion, $qga, $bwlimit);
8118         } else {
8119             if ($dst_drivename eq 'efidisk0') {
8120                 # the relevant data on the efidisk may be smaller than the source
8121                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8122                 # that is given by the OVMF_VARS.fd
8123                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8124                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8125
8126                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8127
8128                 # better for Ceph if block size is not too small, see bug #3324
8129                 my $bs = 1024*1024;
8130
8131                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8132
8133                 if ($src_format eq 'qcow2' && $snapname) {
8134                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8135                         if !min_version(kvm_user_version(), 6, 2);
8136                     push $cmd->@*, '-l', $snapname;
8137                 }
8138                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8139                 run_command($cmd);
8140             } else {
8141                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8142             }
8143         }
8144     }
8145
8146 no_data_clone:
8147     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8148
8149     my $disk = dclone($drive);
8150     delete $disk->{format};
8151     $disk->{file} = $newvolid;
8152     $disk->{size} = $size if defined($size);
8153
8154     return $disk;
8155 }
8156
8157 sub get_running_qemu_version {
8158     my ($vmid) = @_;
8159     my $res = mon_cmd($vmid, "query-version");
8160     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8161 }
8162
8163 sub qemu_use_old_bios_files {
8164     my ($machine_type) = @_;
8165
8166     return if !$machine_type;
8167
8168     my $use_old_bios_files = undef;
8169
8170     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8171         $machine_type = $1;
8172         $use_old_bios_files = 1;
8173     } else {
8174         my $version = extract_version($machine_type, kvm_user_version());
8175         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8176         # load new efi bios files on migration. So this hack is required to allow
8177         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8178         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8179         $use_old_bios_files = !min_version($version, 2, 4);
8180     }
8181
8182     return ($use_old_bios_files, $machine_type);
8183 }
8184
8185 sub get_efivars_size {
8186     my ($conf, $efidisk) = @_;
8187
8188     my $arch = get_vm_arch($conf);
8189     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8190     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8191     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8192     return -s $ovmf_vars;
8193 }
8194
8195 sub update_efidisk_size {
8196     my ($conf) = @_;
8197
8198     return if !defined($conf->{efidisk0});
8199
8200     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8201     $disk->{size} = get_efivars_size($conf);
8202     $conf->{efidisk0} = print_drive($disk);
8203
8204     return;
8205 }
8206
8207 sub update_tpmstate_size {
8208     my ($conf) = @_;
8209
8210     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8211     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8212     $conf->{tpmstate0} = print_drive($disk);
8213 }
8214
8215 sub create_efidisk($$$$$$$) {
8216     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8217
8218     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8219
8220     my $vars_size_b = -s $ovmf_vars;
8221     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8222     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8223     PVE::Storage::activate_volumes($storecfg, [$volid]);
8224
8225     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8226     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8227
8228     return ($volid, $size/1024);
8229 }
8230
8231 sub vm_iothreads_list {
8232     my ($vmid) = @_;
8233
8234     my $res = mon_cmd($vmid, 'query-iothreads');
8235
8236     my $iothreads = {};
8237     foreach my $iothread (@$res) {
8238         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8239     }
8240
8241     return $iothreads;
8242 }
8243
8244 sub scsihw_infos {
8245     my ($conf, $drive) = @_;
8246
8247     my $maxdev = 0;
8248
8249     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8250         $maxdev = 7;
8251     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8252         $maxdev = 1;
8253     } else {
8254         $maxdev = 256;
8255     }
8256
8257     my $controller = int($drive->{index} / $maxdev);
8258     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8259         ? "virtioscsi"
8260         : "scsihw";
8261
8262     return ($maxdev, $controller, $controller_prefix);
8263 }
8264
8265 sub resolve_dst_disk_format {
8266         my ($storecfg, $storeid, $src_volname, $format) = @_;
8267         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8268
8269         if (!$format) {
8270             # if no target format is specified, use the source disk format as hint
8271             if ($src_volname) {
8272                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8273                 $format = qemu_img_format($scfg, $src_volname);
8274             } else {
8275                 return $defFormat;
8276             }
8277         }
8278
8279         # test if requested format is supported - else use default
8280         my $supported = grep { $_ eq $format } @$validFormats;
8281         $format = $defFormat if !$supported;
8282         return $format;
8283 }
8284
8285 # NOTE: if this logic changes, please update docs & possibly gui logic
8286 sub find_vmstate_storage {
8287     my ($conf, $storecfg) = @_;
8288
8289     # first, return storage from conf if set
8290     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8291
8292     my ($target, $shared, $local);
8293
8294     foreach_storage_used_by_vm($conf, sub {
8295         my ($sid) = @_;
8296         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8297         my $dst = $scfg->{shared} ? \$shared : \$local;
8298         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8299     });
8300
8301     # second, use shared storage where VM has at least one disk
8302     # third, use local storage where VM has at least one disk
8303     # fall back to local storage
8304     $target = $shared // $local // 'local';
8305
8306     return $target;
8307 }
8308
8309 sub generate_uuid {
8310     my ($uuid, $uuid_str);
8311     UUID::generate($uuid);
8312     UUID::unparse($uuid, $uuid_str);
8313     return $uuid_str;
8314 }
8315
8316 sub generate_smbios1_uuid {
8317     return "uuid=".generate_uuid();
8318 }
8319
8320 sub nbd_stop {
8321     my ($vmid) = @_;
8322
8323     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8324 }
8325
8326 sub create_reboot_request {
8327     my ($vmid) = @_;
8328     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8329         or die "failed to create reboot trigger file: $!\n";
8330     close($fh);
8331 }
8332
8333 sub clear_reboot_request {
8334     my ($vmid) = @_;
8335     my $path = "/run/qemu-server/$vmid.reboot";
8336     my $res = 0;
8337
8338     $res = unlink($path);
8339     die "could not remove reboot request for $vmid: $!"
8340         if !$res && $! != POSIX::ENOENT;
8341
8342     return $res;
8343 }
8344
8345 sub bootorder_from_legacy {
8346     my ($conf, $bootcfg) = @_;
8347
8348     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8349     my $bootindex_hash = {};
8350     my $i = 1;
8351     foreach my $o (split(//, $boot)) {
8352         $bootindex_hash->{$o} = $i*100;
8353         $i++;
8354     }
8355
8356     my $bootorder = {};
8357
8358     PVE::QemuConfig->foreach_volume($conf, sub {
8359         my ($ds, $drive) = @_;
8360
8361         if (drive_is_cdrom ($drive, 1)) {
8362             if ($bootindex_hash->{d}) {
8363                 $bootorder->{$ds} = $bootindex_hash->{d};
8364                 $bootindex_hash->{d} += 1;
8365             }
8366         } elsif ($bootindex_hash->{c}) {
8367             $bootorder->{$ds} = $bootindex_hash->{c}
8368                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8369             $bootindex_hash->{c} += 1;
8370         }
8371     });
8372
8373     if ($bootindex_hash->{n}) {
8374         for (my $i = 0; $i < $MAX_NETS; $i++) {
8375             my $netname = "net$i";
8376             next if !$conf->{$netname};
8377             $bootorder->{$netname} = $bootindex_hash->{n};
8378             $bootindex_hash->{n} += 1;
8379         }
8380     }
8381
8382     return $bootorder;
8383 }
8384
8385 # Generate default device list for 'boot: order=' property. Matches legacy
8386 # default boot order, but with explicit device names. This is important, since
8387 # the fallback for when neither 'order' nor the old format is specified relies
8388 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8389 sub get_default_bootdevices {
8390     my ($conf) = @_;
8391
8392     my @ret = ();
8393
8394     # harddisk
8395     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8396     push @ret, $first if $first;
8397
8398     # cdrom
8399     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8400     push @ret, $first if $first;
8401
8402     # network
8403     for (my $i = 0; $i < $MAX_NETS; $i++) {
8404         my $netname = "net$i";
8405         next if !$conf->{$netname};
8406         push @ret, $netname;
8407         last;
8408     }
8409
8410     return \@ret;
8411 }
8412
8413 sub device_bootorder {
8414     my ($conf) = @_;
8415
8416     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8417
8418     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8419
8420     my $bootorder = {};
8421     if (!defined($boot) || $boot->{legacy}) {
8422         $bootorder = bootorder_from_legacy($conf, $boot);
8423     } elsif ($boot->{order}) {
8424         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8425         for my $dev (PVE::Tools::split_list($boot->{order})) {
8426             $bootorder->{$dev} = $i++;
8427         }
8428     }
8429
8430     return $bootorder;
8431 }
8432
8433 sub register_qmeventd_handle {
8434     my ($vmid) = @_;
8435
8436     my $fh;
8437     my $peer = "/var/run/qmeventd.sock";
8438     my $count = 0;
8439
8440     for (;;) {
8441         $count++;
8442         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8443         last if $fh;
8444         if ($! != EINTR && $! != EAGAIN) {
8445             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8446         }
8447         if ($count > 4) {
8448             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8449               . "after $count retries\n";
8450         }
8451         usleep(25000);
8452     }
8453
8454     # send handshake to mark VM as backing up
8455     print $fh to_json({vzdump => {vmid => "$vmid"}});
8456
8457     # return handle to be closed later when inhibit is no longer required
8458     return $fh;
8459 }
8460
8461 # bash completion helper
8462
8463 sub complete_backup_archives {
8464     my ($cmdname, $pname, $cvalue) = @_;
8465
8466     my $cfg = PVE::Storage::config();
8467
8468     my $storeid;
8469
8470     if ($cvalue =~ m/^([^:]+):/) {
8471         $storeid = $1;
8472     }
8473
8474     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8475
8476     my $res = [];
8477     foreach my $id (keys %$data) {
8478         foreach my $item (@{$data->{$id}}) {
8479             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8480             push @$res, $item->{volid} if defined($item->{volid});
8481         }
8482     }
8483
8484     return $res;
8485 }
8486
8487 my $complete_vmid_full = sub {
8488     my ($running) = @_;
8489
8490     my $idlist = vmstatus();
8491
8492     my $res = [];
8493
8494     foreach my $id (keys %$idlist) {
8495         my $d = $idlist->{$id};
8496         if (defined($running)) {
8497             next if $d->{template};
8498             next if $running && $d->{status} ne 'running';
8499             next if !$running && $d->{status} eq 'running';
8500         }
8501         push @$res, $id;
8502
8503     }
8504     return $res;
8505 };
8506
8507 sub complete_vmid {
8508     return &$complete_vmid_full();
8509 }
8510
8511 sub complete_vmid_stopped {
8512     return &$complete_vmid_full(0);
8513 }
8514
8515 sub complete_vmid_running {
8516     return &$complete_vmid_full(1);
8517 }
8518
8519 sub complete_storage {
8520
8521     my $cfg = PVE::Storage::config();
8522     my $ids = $cfg->{ids};
8523
8524     my $res = [];
8525     foreach my $sid (keys %$ids) {
8526         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8527         next if !$ids->{$sid}->{content}->{images};
8528         push @$res, $sid;
8529     }
8530
8531     return $res;
8532 }
8533
8534 sub complete_migration_storage {
8535     my ($cmd, $param, $current_value, $all_args) = @_;
8536
8537     my $targetnode = @$all_args[1];
8538
8539     my $cfg = PVE::Storage::config();
8540     my $ids = $cfg->{ids};
8541
8542     my $res = [];
8543     foreach my $sid (keys %$ids) {
8544         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8545         next if !$ids->{$sid}->{content}->{images};
8546         push @$res, $sid;
8547     }
8548
8549     return $res;
8550 }
8551
8552 sub vm_is_paused {
8553     my ($vmid, $include_suspended) = @_;
8554     my $qmpstatus = eval {
8555         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8556         mon_cmd($vmid, "query-status");
8557     };
8558     warn "$@\n" if $@;
8559     return $qmpstatus && (
8560         $qmpstatus->{status} eq "paused" ||
8561         $qmpstatus->{status} eq "prelaunch" ||
8562         ($include_suspended && $qmpstatus->{status} eq "suspended")
8563     );
8564 }
8565
8566 sub check_volume_storage_type {
8567     my ($storecfg, $vol) = @_;
8568
8569     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8570     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8571     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8572
8573     die "storage '$storeid' does not support content-type '$vtype'\n"
8574         if !$scfg->{content}->{$vtype};
8575
8576     return 1;
8577 }
8578
8579 sub add_nets_bridge_fdb {
8580     my ($conf, $vmid) = @_;
8581
8582     for my $opt (keys %$conf) {
8583         next if $opt !~ m/^net(\d+)$/;
8584         my $iface = "tap${vmid}i$1";
8585         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8586         my $net = parse_net($conf->{$opt}, 1) or next;
8587
8588         my $mac = $net->{macaddr};
8589         if (!$mac) {
8590             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8591                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8592             next;
8593         }
8594
8595         my $bridge = $net->{bridge};
8596         if (!$bridge) {
8597             log_warn("Interface '$iface' not attached to any bridge.");
8598             next;
8599         }
8600         if ($have_sdn) {
8601             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8602         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8603             PVE::Network::add_bridge_fdb($iface, $mac);
8604         }
8605     }
8606 }
8607
8608 sub del_nets_bridge_fdb {
8609     my ($conf, $vmid) = @_;
8610
8611     for my $opt (keys %$conf) {
8612         next if $opt !~ m/^net(\d+)$/;
8613         my $iface = "tap${vmid}i$1";
8614
8615         my $net = parse_net($conf->{$opt}) or next;
8616         my $mac = $net->{macaddr} or next;
8617
8618         my $bridge = $net->{bridge};
8619         if ($have_sdn) {
8620             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8621         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8622             PVE::Network::del_bridge_fdb($iface, $mac);
8623         }
8624     }
8625 }
8626
8627 sub create_ifaces_ipams_ips {
8628     my ($conf, $vmid) = @_;
8629
8630     return if !$have_sdn;
8631
8632     foreach my $opt (keys %$conf) {
8633         if ($opt =~ m/^net(\d+)$/) {
8634             my $value = $conf->{$opt};
8635             my $net = PVE::QemuServer::parse_net($value);
8636             eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8637             warn $@ if $@;
8638         }
8639     }
8640 }
8641
8642 sub delete_ifaces_ipams_ips {
8643     my ($conf, $vmid) = @_;
8644
8645     return if !$have_sdn;
8646
8647     foreach my $opt (keys %$conf) {
8648         if ($opt =~ m/^net(\d+)$/) {
8649             my $net = PVE::QemuServer::parse_net($conf->{$opt});
8650             eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8651             warn $@ if $@;
8652         }
8653     }
8654 }
8655
8656 1;