PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive path_is_scsi);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory qw(get_current_memory);
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  62 use PVE::QemuServer::USB;
  63
  64 my $have_sdn;
  65 eval {
  66     require PVE::Network::SDN::Zones;
  67     require PVE::Network::SDN::Vnets;
  68     $have_sdn = 1;
  69 };
  70
  71 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  72 my $OVMF = {
  73     x86_64 => {
  74         '4m-no-smm' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  77         ],
  78         '4m-no-smm-ms' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  81         ],
  82         '4m' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  85         ],
  86         '4m-ms' => [
  87             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  89         ],
  90         # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
  91         # anymore. how can we deperacate this sanely without breaking existing instances, or using
  92         # older backups and snapshot?
  93         default => [
  94             "$EDK2_FW_BASE/OVMF_CODE.fd",
  95             "$EDK2_FW_BASE/OVMF_VARS.fd",
  96         ],
  97     },
  98     aarch64 => {
  99         default => [
 100             "$EDK2_FW_BASE/AAVMF_CODE.fd",
 101             "$EDK2_FW_BASE/AAVMF_VARS.fd",
 102         ],
 103     },
 104 };
 105
 106 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 107
 108 # Note about locking: we use flock on the config file protect against concurent actions.
 109 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 110 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 111 # But you can ignore this kind of lock with the --skiplock flag.
 112
 113 cfs_register_file(
 114     '/qemu-server/',
 115     \&parse_vm_config,
 116     \&write_vm_config
 117 );
 118
 119 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 120     description => "Some command save/restore state from this location.",
 121     type => 'string',
 122     maxLength => 128,
 123     optional => 1,
 124 });
 125
 126 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 127         description => "Specifies the QEMU machine type.",
 128         type => 'string',
 129         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 130         maxLength => 40,
 131         optional => 1,
 132 });
 133
 134 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 135 my $nodename_cache;
 136 sub nodename {
 137     $nodename_cache //= PVE::INotify::nodename();
 138     return $nodename_cache;
 139 }
 140
 141 my $watchdog_fmt = {
 142     model => {
 143         default_key => 1,
 144         type => 'string',
 145         enum => [qw(i6300esb ib700)],
 146         description => "Watchdog type to emulate.",
 147         default => 'i6300esb',
 148         optional => 1,
 149     },
 150     action => {
 151         type => 'string',
 152         enum => [qw(reset shutdown poweroff pause debug none)],
 153         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 154         optional => 1,
 155     },
 156 };
 157 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 158
 159 my $agent_fmt = {
 160     enabled => {
 161         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 162         type => 'boolean',
 163         default => 0,
 164         default_key => 1,
 165     },
 166     fstrim_cloned_disks => {
 167         description => "Run fstrim after moving a disk or migrating the VM.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0,
 171     },
 172     'freeze-fs-on-backup' => {
 173         description => "Freeze/thaw guest filesystems on backup for consistency.",
 174         type => 'boolean',
 175         optional => 1,
 176         default => 1,
 177     },
 178     type => {
 179         description => "Select the agent type",
 180         type => 'string',
 181         default => 'virtio',
 182         optional => 1,
 183         enum => [qw(virtio isa)],
 184     },
 185 };
 186
 187 my $vga_fmt = {
 188     type => {
 189         description => "Select the VGA type.",
 190         type => 'string',
 191         default => 'std',
 192         optional => 1,
 193         default_key => 1,
 194         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 195     },
 196     memory => {
 197         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 198         type => 'integer',
 199         optional => 1,
 200         minimum => 4,
 201         maximum => 512,
 202     },
 203     clipboard => {
 204         description => 'Enable a specific clipboard. If not set, depending on the display type the'
 205             .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
 206         type => 'string',
 207         enum => ['vnc'],
 208         optional => 1,
 209     },
 210 };
 211
 212 my $ivshmem_fmt = {
 213     size => {
 214         type => 'integer',
 215         minimum => 1,
 216         description => "The size of the file in MB.",
 217     },
 218     name => {
 219         type => 'string',
 220         pattern => '[a-zA-Z0-9\-]+',
 221         optional => 1,
 222         format_description => 'string',
 223         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 224     },
 225 };
 226
 227 my $audio_fmt = {
 228     device => {
 229         type => 'string',
 230         enum => [qw(ich9-intel-hda intel-hda AC97)],
 231         description =>  "Configure an audio device."
 232     },
 233     driver =>  {
 234         type => 'string',
 235         enum => ['spice', 'none'],
 236         default => 'spice',
 237         optional => 1,
 238         description => "Driver backend for the audio device."
 239     },
 240 };
 241
 242 my $spice_enhancements_fmt = {
 243     foldersharing => {
 244         type => 'boolean',
 245         optional => 1,
 246         default => '0',
 247         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 248     },
 249     videostreaming =>  {
 250         type => 'string',
 251         enum => ['off', 'all', 'filter'],
 252         default => 'off',
 253         optional => 1,
 254         description => "Enable video streaming. Uses compression for detected video streams."
 255     },
 256 };
 257
 258 my $rng_fmt = {
 259     source => {
 260         type => 'string',
 261         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 262         default_key => 1,
 263         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 264             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 265             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 266             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 267             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 268             ." a hardware RNG from the host.",
 269     },
 270     max_bytes => {
 271         type => 'integer',
 272         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 273             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 274             ." `0` to disable limiting (potentially dangerous!).",
 275         optional => 1,
 276
 277         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 278         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 279         # reading from /dev/urandom
 280         default => 1024,
 281     },
 282     period => {
 283         type => 'integer',
 284         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 285             ." the guest to retrieve another 'max_bytes' of entropy.",
 286         optional => 1,
 287         default => 1000,
 288     },
 289 };
 290
 291 my $meta_info_fmt = {
 292     'ctime' => {
 293         type => 'integer',
 294         description => "The guest creation timestamp as UNIX epoch time",
 295         minimum => 0,
 296         optional => 1,
 297     },
 298     'creation-qemu' => {
 299         type => 'string',
 300         description => "The QEMU (machine) version from the time this VM was created.",
 301         pattern => '\d+(\.\d+)+',
 302         optional => 1,
 303     },
 304 };
 305
 306 my $confdesc = {
 307     onboot => {
 308         optional => 1,
 309         type => 'boolean',
 310         description => "Specifies whether a VM will be started during system bootup.",
 311         default => 0,
 312     },
 313     autostart => {
 314         optional => 1,
 315         type => 'boolean',
 316         description => "Automatic restart after crash (currently ignored).",
 317         default => 0,
 318     },
 319     hotplug => {
 320         optional => 1,
 321         type => 'string', format => 'pve-hotplug-features',
 322         description => "Selectively enable hotplug features. This is a comma separated list of"
 323             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 324             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 325             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 326             ." windows > 7.",
 327         default => 'network,disk,usb',
 328     },
 329     reboot => {
 330         optional => 1,
 331         type => 'boolean',
 332         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 333         default => 1,
 334     },
 335     lock => {
 336         optional => 1,
 337         type => 'string',
 338         description => "Lock/unlock the VM.",
 339         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 340     },
 341     cpulimit => {
 342         optional => 1,
 343         type => 'number',
 344         description => "Limit of CPU usage.",
 345         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 346             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 347         minimum => 0,
 348         maximum => 128,
 349         default => 0,
 350     },
 351     cpuunits => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 355         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 356             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 357             ." weights of all the other running VMs.",
 358         minimum => 1,
 359         maximum => 262144,
 360         default => 'cgroup v1: 1024, cgroup v2: 100',
 361     },
 362     memory => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Memory properties.",
 366         format => $PVE::QemuServer::Memory::memory_fmt
 367     },
 368     balloon => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 372         minimum => 0,
 373     },
 374     shares => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 378             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 379             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 380         minimum => 0,
 381         maximum => 50000,
 382         default => 1000,
 383     },
 384     keyboard => {
 385         optional => 1,
 386         type => 'string',
 387         description => "Keyboard layout for VNC server. This option is generally not required and"
 388         ." is often better handled from within the guest OS.",
 389         enum => PVE::Tools::kvmkeymaplist(),
 390         default => undef,
 391     },
 392     name => {
 393         optional => 1,
 394         type => 'string', format => 'dns-name',
 395         description => "Set a name for the VM. Only used on the configuration web interface.",
 396     },
 397     scsihw => {
 398         optional => 1,
 399         type => 'string',
 400         description => "SCSI controller model",
 401         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 402         default => 'lsi',
 403     },
 404     description => {
 405         optional => 1,
 406         type => 'string',
 407         description => "Description for the VM. Shown in the web-interface VM's summary."
 408             ." This is saved as comment inside the configuration file.",
 409         maxLength => 1024 * 8,
 410     },
 411     ostype => {
 412         optional => 1,
 413         type => 'string',
 414         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 415         description => "Specify guest operating system.",
 416         verbose_description => <<EODESC,
 417 Specify guest operating system. This is used to enable special
 418 optimization/features for specific operating systems:
 419
 420 [horizontal]
 421 other;; unspecified OS
 422 wxp;; Microsoft Windows XP
 423 w2k;; Microsoft Windows 2000
 424 w2k3;; Microsoft Windows 2003
 425 w2k8;; Microsoft Windows 2008
 426 wvista;; Microsoft Windows Vista
 427 win7;; Microsoft Windows 7
 428 win8;; Microsoft Windows 8/2012/2012r2
 429 win10;; Microsoft Windows 10/2016/2019
 430 win11;; Microsoft Windows 11/2022
 431 l24;; Linux 2.4 Kernel
 432 l26;; Linux 2.6 - 6.X Kernel
 433 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 434 EODESC
 435     },
 436     boot => {
 437         optional => 1,
 438         type => 'string', format => 'pve-qm-boot',
 439         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 440             ." key or 'legacy=' is deprecated.",
 441     },
 442     bootdisk => {
 443         optional => 1,
 444         type => 'string', format => 'pve-qm-bootdisk',
 445         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 446         pattern => '(ide|sata|scsi|virtio)\d+',
 447     },
 448     smp => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "The number of CPUs. Please use option -sockets instead.",
 452         minimum => 1,
 453         default => 1,
 454     },
 455     sockets => {
 456         optional => 1,
 457         type => 'integer',
 458         description => "The number of CPU sockets.",
 459         minimum => 1,
 460         default => 1,
 461     },
 462     cores => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "The number of cores per socket.",
 466         minimum => 1,
 467         default => 1,
 468     },
 469     numa => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable NUMA.",
 473         default => 0,
 474     },
 475     hugepages => {
 476         optional => 1,
 477         type => 'string',
 478         description => "Enable/disable hugepages memory.",
 479         enum => [qw(any 2 1024)],
 480     },
 481     keephugepages => {
 482         optional => 1,
 483         type => 'boolean',
 484         default => 0,
 485         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 486             ." after VM shutdown and can be used for subsequent starts.",
 487     },
 488     vcpus => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Number of hotplugged vcpus.",
 492         minimum => 1,
 493         default => 0,
 494     },
 495     acpi => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable ACPI.",
 499         default => 1,
 500     },
 501     agent => {
 502         optional => 1,
 503         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 504         type => 'string',
 505         format => $agent_fmt,
 506     },
 507     kvm => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable KVM hardware virtualization.",
 511         default => 1,
 512     },
 513     tdf => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable time drift fix.",
 517         default => 0,
 518     },
 519     localtime => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 523             ." the `ostype` indicates a Microsoft Windows OS.",
 524     },
 525     freeze => {
 526         optional => 1,
 527         type => 'boolean',
 528         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 529     },
 530     vga => {
 531         optional => 1,
 532         type => 'string', format => $vga_fmt,
 533         description => "Configure the VGA hardware.",
 534         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 535             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 536             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 537             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 538             ." display server. For win* OS you can select how many independent displays you want,"
 539             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 540             ." using a serial device as terminal.",
 541     },
 542     watchdog => {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-watchdog',
 545         description => "Create a virtual hardware watchdog device.",
 546         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 547             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 548             ." else the watchdog will reset the guest (or execute the respective action specified)",
 549     },
 550     startdate => {
 551         optional => 1,
 552         type => 'string',
 553         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 554         description => "Set the initial date of the real time clock. Valid format for date are:"
 555             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 556         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 557         default => 'now',
 558     },
 559     startup =>  get_standard_option('pve-startup-order'),
 560     template => {
 561         optional => 1,
 562         type => 'boolean',
 563         description => "Enable/disable Template.",
 564         default => 0,
 565     },
 566     args => {
 567         optional => 1,
 568         type => 'string',
 569         description => "Arbitrary arguments passed to kvm.",
 570         verbose_description => <<EODESCR,
 571 Arbitrary arguments passed to kvm, for example:
 572
 573 args: -no-reboot -smbios 'type=0,vendor=FOO'
 574
 575 NOTE: this option is for experts only.
 576 EODESCR
 577     },
 578     tablet => {
 579         optional => 1,
 580         type => 'boolean',
 581         default => 1,
 582         description => "Enable/disable the USB tablet device.",
 583         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 584             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 585             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 586             ." may consider disabling this to save some context switches. This is turned off by"
 587             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 588     },
 589     migrate_speed => {
 590         optional => 1,
 591         type => 'integer',
 592         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 593         minimum => 0,
 594         default => 0,
 595     },
 596     migrate_downtime => {
 597         optional => 1,
 598         type => 'number',
 599         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 600         minimum => 0,
 601         default => 0.1,
 602     },
 603     cdrom => {
 604         optional => 1,
 605         type => 'string', format => 'pve-qm-ide',
 606         typetext => '<volume>',
 607         description => "This is an alias for option -ide2",
 608     },
 609     cpu => {
 610         optional => 1,
 611         description => "Emulated CPU type.",
 612         type => 'string',
 613         format => 'pve-vm-cpu-conf',
 614     },
 615     parent => get_standard_option('pve-snapshot-name', {
 616         optional => 1,
 617         description => "Parent snapshot name. This is used internally, and should not be modified.",
 618     }),
 619     snaptime => {
 620         optional => 1,
 621         description => "Timestamp for snapshots.",
 622         type => 'integer',
 623         minimum => 0,
 624     },
 625     vmstate => {
 626         optional => 1,
 627         type => 'string', format => 'pve-volume-id',
 628         description => "Reference to a volume which stores the VM state. This is used internally"
 629             ." for snapshots.",
 630     },
 631     vmstatestorage => get_standard_option('pve-storage-id', {
 632         description => "Default storage for VM state volumes/files.",
 633         optional => 1,
 634     }),
 635     runningmachine => get_standard_option('pve-qemu-machine', {
 636         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 637             ." for snapshots.",
 638     }),
 639     runningcpu => {
 640         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 641             ." internally for snapshots.",
 642         optional => 1,
 643         type => 'string',
 644         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 645         format_description => 'QEMU -cpu parameter'
 646     },
 647     machine => get_standard_option('pve-qemu-machine'),
 648     arch => {
 649         description => "Virtual processor architecture. Defaults to the host.",
 650         optional => 1,
 651         type => 'string',
 652         enum => [qw(x86_64 aarch64)],
 653     },
 654     smbios1 => {
 655         description => "Specify SMBIOS type 1 fields.",
 656         type => 'string', format => 'pve-qm-smbios1',
 657         maxLength => 512,
 658         optional => 1,
 659     },
 660     protection => {
 661         optional => 1,
 662         type => 'boolean',
 663         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 664             ." remove disk operations.",
 665         default => 0,
 666     },
 667     bios => {
 668         optional => 1,
 669         type => 'string',
 670         enum => [ qw(seabios ovmf) ],
 671         description => "Select BIOS implementation.",
 672         default => 'seabios',
 673     },
 674     vmgenid => {
 675         type => 'string',
 676         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 677         format_description => 'UUID',
 678         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 679             ." to disable explicitly.",
 680         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 681             ." value identifier to the guest OS. This allows to notify the guest operating system"
 682             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 683             ." execution or creation from a template). The guest operating system notices the"
 684             ." change, and is then able to react as appropriate by marking its copies of"
 685             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 686             ."Note that auto-creation only works when done through API/CLI create or update methods"
 687             .", but not when manually editing the config file.",
 688         default => "1 (autogenerated)",
 689         optional => 1,
 690     },
 691     hookscript => {
 692         type => 'string',
 693         format => 'pve-volume-id',
 694         optional => 1,
 695         description => "Script that will be executed during various steps in the vms lifetime.",
 696     },
 697     ivshmem => {
 698         type => 'string',
 699         format => $ivshmem_fmt,
 700         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 701             ." the host.",
 702         optional => 1,
 703     },
 704     audio0 => {
 705         type => 'string',
 706         format => $audio_fmt,
 707         description => "Configure a audio device, useful in combination with QXL/Spice.",
 708         optional => 1
 709     },
 710     spice_enhancements => {
 711         type => 'string',
 712         format => $spice_enhancements_fmt,
 713         description => "Configure additional enhancements for SPICE.",
 714         optional => 1
 715     },
 716     tags => {
 717         type => 'string', format => 'pve-tag-list',
 718         description => 'Tags of the VM. This is only meta information.',
 719         optional => 1,
 720     },
 721     rng0 => {
 722         type => 'string',
 723         format => $rng_fmt,
 724         description => "Configure a VirtIO-based Random Number Generator.",
 725         optional => 1,
 726     },
 727     meta => {
 728         type => 'string',
 729         format => $meta_info_fmt,
 730         description => "Some (read-only) meta-information about this guest.",
 731         optional => 1,
 732     },
 733     affinity => {
 734         type => 'string', format => 'pve-cpuset',
 735         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 736         optional => 1,
 737     },
 738 };
 739
 740 my $cicustom_fmt = {
 741     meta => {
 742         type => 'string',
 743         optional => 1,
 744         description => 'Specify a custom file containing all meta data passed to the VM via"
 745             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 746         format => 'pve-volume-id',
 747         format_description => 'volume',
 748     },
 749     network => {
 750         type => 'string',
 751         optional => 1,
 752         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 753         format => 'pve-volume-id',
 754         format_description => 'volume',
 755     },
 756     user => {
 757         type => 'string',
 758         optional => 1,
 759         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 760         format => 'pve-volume-id',
 761         format_description => 'volume',
 762     },
 763     vendor => {
 764         type => 'string',
 765         optional => 1,
 766         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 767         format => 'pve-volume-id',
 768         format_description => 'volume',
 769     },
 770 };
 771 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 772
 773 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 774 my $confdesc_cloudinit = {
 775     citype => {
 776         optional => 1,
 777         type => 'string',
 778         description => 'Specifies the cloud-init configuration format. The default depends on the'
 779             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 780             .' and `configdrive2` for windows.',
 781         enum => ['configdrive2', 'nocloud', 'opennebula'],
 782     },
 783     ciuser => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: User name to change ssh keys and password for instead of the"
 787             ." image's configured default user.",
 788     },
 789     cipassword => {
 790         optional => 1,
 791         type => 'string',
 792         description => 'cloud-init: Password to assign the user. Using this is generally not'
 793             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 794             .' support hashed passwords.',
 795     },
 796     ciupgrade => {
 797         optional => 1,
 798         type => 'boolean',
 799         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 800         default => 1,
 801     },
 802     cicustom => {
 803         optional => 1,
 804         type => 'string',
 805         description => 'cloud-init: Specify custom files to replace the automatically generated'
 806             .' ones at start.',
 807         format => 'pve-qm-cicustom',
 808     },
 809     searchdomain => {
 810         optional => 1,
 811         type => 'string',
 812         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 813             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 814             .' are set.',
 815     },
 816     nameserver => {
 817         optional => 1,
 818         type => 'string', format => 'address-list',
 819         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 820             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 821             .' are set.',
 822     },
 823     sshkeys => {
 824         optional => 1,
 825         type => 'string',
 826         format => 'urlencoded',
 827         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 828     },
 829 };
 830
 831 # what about other qemu settings ?
 832 #cpu => 'string',
 833 #machine => 'string',
 834 #fda => 'file',
 835 #fdb => 'file',
 836 #mtdblock => 'file',
 837 #sd => 'file',
 838 #pflash => 'file',
 839 #snapshot => 'bool',
 840 #bootp => 'file',
 841 ##tftp => 'dir',
 842 ##smb => 'dir',
 843 #kernel => 'file',
 844 #append => 'string',
 845 #initrd => 'file',
 846 ##soundhw => 'string',
 847
 848 while (my ($k, $v) = each %$confdesc) {
 849     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 850 }
 851
 852 my $MAX_NETS = 32;
 853 my $MAX_SERIAL_PORTS = 4;
 854 my $MAX_PARALLEL_PORTS = 3;
 855
 856 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 857     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 858 }
 859
 860 my $nic_model_list = [
 861     'e1000',
 862     'e1000-82540em',
 863     'e1000-82544gc',
 864     'e1000-82545em',
 865     'e1000e',
 866     'i82551',
 867     'i82557b',
 868     'i82559er',
 869     'ne2k_isa',
 870     'ne2k_pci',
 871     'pcnet',
 872     'rtl8139',
 873     'virtio',
 874     'vmxnet3',
 875 ];
 876 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 877
 878 my $net_fmt_bridge_descr = <<__EOD__;
 879 Bridge to attach the network device to. The Proxmox VE standard bridge
 880 is called 'vmbr0'.
 881
 882 If you do not specify a bridge, we create a kvm user (NATed) network
 883 device, which provides DHCP and DNS services. The following addresses
 884 are used:
 885
 886  10.0.2.2   Gateway
 887  10.0.2.3   DNS Server
 888  10.0.2.4   SMB Server
 889
 890 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 891 __EOD__
 892
 893 my $net_fmt = {
 894     macaddr  => get_standard_option('mac-addr', {
 895         description => "MAC address. That address must be unique withing your network. This is"
 896             ." automatically generated if not specified.",
 897     }),
 898     model => {
 899         type => 'string',
 900         description => "Network Card Model. The 'virtio' model provides the best performance with"
 901             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 902             ." best to use 'e1000'.",
 903         enum => $nic_model_list,
 904         default_key => 1,
 905     },
 906     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 907     bridge => get_standard_option('pve-bridge-id', {
 908         description => $net_fmt_bridge_descr,
 909         optional => 1,
 910     }),
 911     queues => {
 912         type => 'integer',
 913         minimum => 0, maximum => 64,
 914         description => 'Number of packet queues to be used on the device.',
 915         optional => 1,
 916     },
 917     rate => {
 918         type => 'number',
 919         minimum => 0,
 920         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 921         optional => 1,
 922     },
 923     tag => {
 924         type => 'integer',
 925         minimum => 1, maximum => 4094,
 926         description => 'VLAN tag to apply to packets on this interface.',
 927         optional => 1,
 928     },
 929     trunks => {
 930         type => 'string',
 931         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 932         description => 'VLAN trunks to pass through this interface.',
 933         format_description => 'vlanid[;vlanid...]',
 934         optional => 1,
 935     },
 936     firewall => {
 937         type => 'boolean',
 938         description => 'Whether this interface should be protected by the firewall.',
 939         optional => 1,
 940     },
 941     link_down => {
 942         type => 'boolean',
 943         description => 'Whether this interface should be disconnected (like pulling the plug).',
 944         optional => 1,
 945     },
 946     mtu => {
 947         type => 'integer',
 948         minimum => 1, maximum => 65520,
 949         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 950         optional => 1,
 951     },
 952 };
 953
 954 my $netdesc = {
 955     optional => 1,
 956     type => 'string', format => $net_fmt,
 957     description => "Specify network devices.",
 958 };
 959
 960 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 961
 962 my $ipconfig_fmt = {
 963     ip => {
 964         type => 'string',
 965         format => 'pve-ipv4-config',
 966         format_description => 'IPv4Format/CIDR',
 967         description => 'IPv4 address in CIDR format.',
 968         optional => 1,
 969         default => 'dhcp',
 970     },
 971     gw => {
 972         type => 'string',
 973         format => 'ipv4',
 974         format_description => 'GatewayIPv4',
 975         description => 'Default gateway for IPv4 traffic.',
 976         optional => 1,
 977         requires => 'ip',
 978     },
 979     ip6 => {
 980         type => 'string',
 981         format => 'pve-ipv6-config',
 982         format_description => 'IPv6Format/CIDR',
 983         description => 'IPv6 address in CIDR format.',
 984         optional => 1,
 985         default => 'dhcp',
 986     },
 987     gw6 => {
 988         type => 'string',
 989         format => 'ipv6',
 990         format_description => 'GatewayIPv6',
 991         description => 'Default gateway for IPv6 traffic.',
 992         optional => 1,
 993         requires => 'ip6',
 994     },
 995 };
 996 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 997 my $ipconfigdesc = {
 998     optional => 1,
 999     type => 'string', format => 'pve-qm-ipconfig',
1000     description => <<'EODESCR',
1001 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1002
1003 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1004
1005 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1006 gateway should be provided.
1007 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1008 cloud-init 19.4 or newer.
1009
1010 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1011 dhcp on IPv4.
1012 EODESCR
1013 };
1014 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1015
1016 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1017     $confdesc->{"net$i"} = $netdesc;
1018     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1019 }
1020
1021 foreach my $key (keys %$confdesc_cloudinit) {
1022     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1023 }
1024
1025 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1026 sub pve_verify_cpuset {
1027     my ($set_text, $noerr) = @_;
1028
1029     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1030
1031     if ($@) {
1032         return if $noerr;
1033         die "unable to parse cpuset option\n";
1034     }
1035
1036     return PVE::CpuSet->new($members)->short_string();
1037 }
1038
1039 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1040 sub verify_volume_id_or_qm_path {
1041     my ($volid, $noerr) = @_;
1042
1043     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1044
1045     return verify_volume_id_or_absolute_path($volid, $noerr);
1046 }
1047
1048 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1049 sub verify_volume_id_or_absolute_path {
1050     my ($volid, $noerr) = @_;
1051
1052     return $volid if $volid =~ m|^/|;
1053
1054     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1055     if ($@) {
1056         return if $noerr;
1057         die $@;
1058     }
1059     return $volid;
1060 }
1061
1062 my $serialdesc = {
1063         optional => 1,
1064         type => 'string',
1065         pattern => '(/dev/.+|socket)',
1066         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1067         verbose_description =>  <<EODESCR,
1068 Create a serial device inside the VM (n is 0 to 3), and pass through a
1069 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1070 host side (use 'qm terminal' to open a terminal connection).
1071
1072 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1073 use with special care.
1074
1075 CAUTION: Experimental! User reported problems with this option.
1076 EODESCR
1077 };
1078
1079 my $paralleldesc= {
1080         optional => 1,
1081         type => 'string',
1082         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1083         description =>  "Map host parallel devices (n is 0 to 2).",
1084         verbose_description =>  <<EODESCR,
1085 Map host parallel devices (n is 0 to 2).
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1095     $confdesc->{"parallel$i"} = $paralleldesc;
1096 }
1097
1098 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1099     $confdesc->{"serial$i"} = $serialdesc;
1100 }
1101
1102 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1103     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1104 }
1105
1106 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1107     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1108 }
1109
1110 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1111     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1112 }
1113
1114 my $boot_fmt = {
1115     legacy => {
1116         optional => 1,
1117         default_key => 1,
1118         type => 'string',
1119         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1120                      . " Deprecated, use 'order=' instead.",
1121         pattern => '[acdn]{1,4}',
1122         format_description => "[acdn]{1,4}",
1123
1124         # note: this is also the fallback if boot: is not given at all
1125         default => 'cdn',
1126     },
1127     order => {
1128         optional => 1,
1129         type => 'string',
1130         format => 'pve-qm-bootdev-list',
1131         format_description => "device[;device...]",
1132         description => <<EODESC,
1133 The guest will attempt to boot from devices in the order they appear here.
1134
1135 Disks, optical drives and passed-through storage USB devices will be directly
1136 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1137 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1138
1139 Note that only devices in this list will be marked as bootable and thus loaded
1140 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1141 (e.g. software-raid), you need to specify all of them here.
1142
1143 Overrides the deprecated 'legacy=[acdn]*' value when given.
1144 EODESC
1145     },
1146 };
1147 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1148
1149 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1150 sub verify_bootdev {
1151     my ($dev, $noerr) = @_;
1152
1153     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1154     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1155
1156     my $check = sub {
1157         my ($base) = @_;
1158         return 0 if $dev !~ m/^$base\d+$/;
1159         return 0 if !$confdesc->{$dev};
1160         return 1;
1161     };
1162
1163     return $dev if $check->("net");
1164     return $dev if $check->("usb");
1165     return $dev if $check->("hostpci");
1166
1167     return if $noerr;
1168     die "invalid boot device '$dev'\n";
1169 }
1170
1171 sub print_bootorder {
1172     my ($devs) = @_;
1173     return "" if !@$devs;
1174     my $data = { order => join(';', @$devs) };
1175     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1176 }
1177
1178 my $kvm_api_version = 0;
1179
1180 sub kvm_version {
1181     return $kvm_api_version if $kvm_api_version;
1182
1183     open my $fh, '<', '/dev/kvm' or return;
1184
1185     # 0xae00 => KVM_GET_API_VERSION
1186     $kvm_api_version = ioctl($fh, 0xae00, 0);
1187     close($fh);
1188
1189     return $kvm_api_version;
1190 }
1191
1192 my $kvm_user_version = {};
1193 my $kvm_mtime = {};
1194
1195 sub kvm_user_version {
1196     my ($binary) = @_;
1197
1198     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1199     my $st = stat($binary);
1200
1201     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1202     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1203         $cachedmtime == $st->mtime;
1204
1205     $kvm_user_version->{$binary} = 'unknown';
1206     $kvm_mtime->{$binary} = $st->mtime;
1207
1208     my $code = sub {
1209         my $line = shift;
1210         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1211             $kvm_user_version->{$binary} = $2;
1212         }
1213     };
1214
1215     eval { run_command([$binary, '--version'], outfunc => $code); };
1216     warn $@ if $@;
1217
1218     return $kvm_user_version->{$binary};
1219
1220 }
1221 my sub extract_version {
1222     my ($machine_type, $version) = @_;
1223     $version = kvm_user_version() if !defined($version);
1224     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1225 }
1226
1227 sub kernel_has_vhost_net {
1228     return -c '/dev/vhost-net';
1229 }
1230
1231 sub option_exists {
1232     my $key = shift;
1233     return defined($confdesc->{$key});
1234 }
1235
1236 my $cdrom_path;
1237 sub get_cdrom_path {
1238
1239     return $cdrom_path if defined($cdrom_path);
1240
1241     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1242
1243     if (!defined($cdrom_path)) {
1244         log_warn("no physical CD-ROM available, ignoring");
1245         $cdrom_path = '';
1246     }
1247
1248     return $cdrom_path;
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269      if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1314             if !$vtype;
1315         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1316         verify_media_type($opt, $vtype, $drive->{media});
1317         $drive->{file} = $volid;
1318     }
1319
1320     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1321 }
1322
1323 sub parse_hotplug_features {
1324     my ($data) = @_;
1325
1326     my $res = {};
1327
1328     return $res if $data eq '0';
1329
1330     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1331
1332     foreach my $feature (PVE::Tools::split_list($data)) {
1333         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1334             $res->{$1} = 1;
1335         } else {
1336             die "invalid hotplug feature '$feature'\n";
1337         }
1338     }
1339     return $res;
1340 }
1341
1342 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1343 sub pve_verify_hotplug_features {
1344     my ($value, $noerr) = @_;
1345
1346     return $value if parse_hotplug_features($value);
1347
1348     return if $noerr;
1349
1350     die "unable to parse hotplug option\n";
1351 }
1352
1353 sub assert_clipboard_config {
1354     my ($vga) = @_;
1355
1356     my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1357
1358     if (
1359         $vga->{'clipboard'}
1360         && $vga->{'clipboard'} eq 'vnc'
1361         && $vga->{type}
1362         && $vga->{type} !~ $clipboard_regex
1363     ) {
1364         die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1365     }
1366 }
1367
1368 sub print_tabletdevice_full {
1369     my ($conf, $arch) = @_;
1370
1371     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1372
1373     # we use uhci for old VMs because tablet driver was buggy in older qemu
1374     my $usbbus;
1375     if ($q35 || $arch eq 'aarch64') {
1376         $usbbus = 'ehci';
1377     } else {
1378         $usbbus = 'uhci';
1379     }
1380
1381     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1382 }
1383
1384 sub print_keyboarddevice_full {
1385     my ($conf, $arch) = @_;
1386
1387     return if $arch ne 'aarch64';
1388
1389     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1390 }
1391
1392 my sub get_drive_id {
1393     my ($drive) = @_;
1394     return "$drive->{interface}$drive->{index}";
1395 }
1396
1397 sub print_drivedevice_full {
1398     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1399
1400     my $device = '';
1401     my $maxdev = 0;
1402
1403     my $drive_id = get_drive_id($drive);
1404     if ($drive->{interface} eq 'virtio') {
1405         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1406         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1407         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1408     } elsif ($drive->{interface} eq 'scsi') {
1409
1410         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1411         my $unit = $drive->{index} % $maxdev;
1412         my $devicetype = 'hd';
1413         my $path = '';
1414         if (drive_is_cdrom($drive)) {
1415             $devicetype = 'cd';
1416         } else {
1417             if ($drive->{file} =~ m|^/|) {
1418                 $path = $drive->{file};
1419                 if (my $info = path_is_scsi($path)) {
1420                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1421                         $devicetype = 'block';
1422                     } elsif ($info->{type} == 1) { # tape
1423                         $devicetype = 'generic';
1424                     }
1425                 }
1426             } else {
1427                  $path = PVE::Storage::path($storecfg, $drive->{file});
1428             }
1429
1430             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1431             my $version = extract_version($machine_type, kvm_user_version());
1432             if ($path =~ m/^iscsi\:\/\// &&
1433                !min_version($version, 4, 1)) {
1434                 $devicetype = 'generic';
1435             }
1436         }
1437
1438         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1439             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1440         } else {
1441             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1442                 .",lun=$drive->{index}";
1443         }
1444         $device .= ",drive=drive-$drive_id,id=$drive_id";
1445
1446         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1447             $device .= ",rotation_rate=1";
1448         }
1449         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1450
1451     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1452         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1453         my $controller = int($drive->{index} / $maxdev);
1454         my $unit = $drive->{index} % $maxdev;
1455
1456         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1457         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1458         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1459         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1460         # were before. Move odd ones up by 2 where they don't clash.
1461         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1462             $controller += 2 * ($unit % 2);
1463             $unit = 0;
1464         }
1465
1466         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1467
1468         $device = "ide-$devicetype";
1469         if ($drive->{interface} eq 'ide') {
1470             $device .= ",bus=ide.$controller,unit=$unit";
1471         } else {
1472             $device .= ",bus=ahci$controller.$unit";
1473         }
1474         $device .= ",drive=drive-$drive_id,id=$drive_id";
1475
1476         if ($devicetype eq 'hd') {
1477             if (my $model = $drive->{model}) {
1478                 $model = URI::Escape::uri_unescape($model);
1479                 $device .= ",model=$model";
1480             }
1481             if ($drive->{ssd}) {
1482                 $device .= ",rotation_rate=1";
1483             }
1484         }
1485         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1486     } elsif ($drive->{interface} eq 'usb') {
1487         die "implement me";
1488         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1489     } else {
1490         die "unsupported interface type";
1491     }
1492
1493     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1494
1495     if (my $serial = $drive->{serial}) {
1496         $serial = URI::Escape::uri_unescape($serial);
1497         $device .= ",serial=$serial";
1498     }
1499
1500
1501     return $device;
1502 }
1503
1504 sub get_initiator_name {
1505     my $initiator;
1506
1507     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1508     while (defined(my $line = <$fh>)) {
1509         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1510         $initiator = $1;
1511         last;
1512     }
1513     $fh->close();
1514
1515     return $initiator;
1516 }
1517
1518 my sub storage_allows_io_uring_default {
1519     my ($scfg, $cache_direct) = @_;
1520
1521     # io_uring with cache mode writeback or writethrough on krbd will hang...
1522     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1523
1524     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1525     # sometimes, just plain disable...
1526     return if $scfg && $scfg->{type} eq 'lvm';
1527
1528     # io_uring causes problems when used with CIFS since kernel 5.15
1529     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1530     return if $scfg && $scfg->{type} eq 'cifs';
1531
1532     return 1;
1533 }
1534
1535 my sub drive_uses_cache_direct {
1536     my ($drive, $scfg) = @_;
1537
1538     my $cache_direct = 0;
1539
1540     if (my $cache = $drive->{cache}) {
1541         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1542     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1543         $cache_direct = 1;
1544     }
1545
1546     return $cache_direct;
1547 }
1548
1549 sub print_drive_commandline_full {
1550     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1551
1552     my $path;
1553     my $volid = $drive->{file};
1554     my $format = $drive->{format};
1555     my $drive_id = get_drive_id($drive);
1556
1557     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1558     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1559
1560     if (drive_is_cdrom($drive)) {
1561         $path = get_iso_path($storecfg, $vmid, $volid);
1562         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1563     } else {
1564         if ($storeid) {
1565             $path = PVE::Storage::path($storecfg, $volid);
1566             $format //= qemu_img_format($scfg, $volname);
1567         } else {
1568             $path = $volid;
1569             $format //= "raw";
1570         }
1571    }
1572
1573    my $is_rbd = $path =~ m/^rbd:/;
1574
1575     my $opts = '';
1576     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1577     foreach my $o (@qemu_drive_options) {
1578         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1579     }
1580
1581     # snapshot only accepts on|off
1582     if (defined($drive->{snapshot})) {
1583         my $v = $drive->{snapshot} ? 'on' : 'off';
1584         $opts .= ",snapshot=$v";
1585     }
1586
1587     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1588         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1589     }
1590
1591     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1592         my ($dir, $qmpname) = @$type;
1593         if (my $v = $drive->{"mbps$dir"}) {
1594             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1595         }
1596         if (my $v = $drive->{"mbps${dir}_max"}) {
1597             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1598         }
1599         if (my $v = $drive->{"bps${dir}_max_length"}) {
1600             $opts .= ",throttling.bps$qmpname-max-length=$v";
1601         }
1602         if (my $v = $drive->{"iops${dir}"}) {
1603             $opts .= ",throttling.iops$qmpname=$v";
1604         }
1605         if (my $v = $drive->{"iops${dir}_max"}) {
1606             $opts .= ",throttling.iops$qmpname-max=$v";
1607         }
1608         if (my $v = $drive->{"iops${dir}_max_length"}) {
1609             $opts .= ",throttling.iops$qmpname-max-length=$v";
1610         }
1611     }
1612
1613     if ($pbs_name) {
1614         $format = "rbd" if $is_rbd;
1615         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1616             if !$format;
1617         $opts .= ",format=alloc-track,file.driver=$format";
1618     } elsif ($format) {
1619         $opts .= ",format=$format";
1620     }
1621
1622     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1623
1624     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1625
1626     if (!$drive->{aio}) {
1627         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1628             # io_uring supports all cache modes
1629             $opts .= ",aio=io_uring";
1630         } else {
1631             # aio native works only with O_DIRECT
1632             if($cache_direct) {
1633                 $opts .= ",aio=native";
1634             } else {
1635                 $opts .= ",aio=threads";
1636             }
1637         }
1638     }
1639
1640     if (!drive_is_cdrom($drive)) {
1641         my $detectzeroes;
1642         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1643             $detectzeroes = 'off';
1644         } elsif ($drive->{discard}) {
1645             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1646         } else {
1647             # This used to be our default with discard not being specified:
1648             $detectzeroes = 'on';
1649         }
1650
1651         # note: 'detect-zeroes' works per blockdev and we want it to persist
1652         # after the alloc-track is removed, so put it on 'file' directly
1653         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1654         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1655     }
1656
1657     if ($pbs_name) {
1658         $opts .= ",backing=$pbs_name";
1659         $opts .= ",auto-remove=on";
1660     }
1661
1662     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1663     my $file_param = "file";
1664     if ($pbs_name) {
1665         # non-rbd drivers require the underlying file to be a seperate block
1666         # node, so add a second .file indirection
1667         $file_param .= ".file" if !$is_rbd;
1668         $file_param .= ".filename";
1669     }
1670     my $pathinfo = $path ? "$file_param=$path," : '';
1671
1672     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1673 }
1674
1675 sub print_pbs_blockdev {
1676     my ($pbs_conf, $pbs_name) = @_;
1677     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1678     $blockdev .= ",repository=$pbs_conf->{repository}";
1679     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1680     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1681     $blockdev .= ",archive=$pbs_conf->{archive}";
1682     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1683     return $blockdev;
1684 }
1685
1686 sub print_netdevice_full {
1687     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1688
1689     my $device = $net->{model};
1690     if ($net->{model} eq 'virtio') {
1691          $device = 'virtio-net-pci';
1692      };
1693
1694     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1695     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1696     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1697         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1698         # and out of each queue plus one config interrupt and control vector queue
1699         my $vectors = $net->{queues} * 2 + 2;
1700         $tmpstr .= ",vectors=$vectors,mq=on";
1701         if (min_version($machine_version, 7, 1)) {
1702             $tmpstr .= ",packed=on";
1703         }
1704     }
1705
1706     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1707         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1708     }
1709
1710     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1711
1712     if (my $mtu = $net->{mtu}) {
1713         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1714             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1715             if ($mtu == 1) {
1716                  $mtu = $bridge_mtu;
1717             } elsif ($mtu < 576) {
1718                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1719             } elsif ($mtu > $bridge_mtu) {
1720                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1721             }
1722             $tmpstr .= ",host_mtu=$mtu";
1723         } else {
1724             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1725         }
1726     }
1727
1728     if ($use_old_bios_files) {
1729         my $romfile;
1730         if ($device eq 'virtio-net-pci') {
1731             $romfile = 'pxe-virtio.rom';
1732         } elsif ($device eq 'e1000') {
1733             $romfile = 'pxe-e1000.rom';
1734         } elsif ($device eq 'e1000e') {
1735             $romfile = 'pxe-e1000e.rom';
1736         } elsif ($device eq 'ne2k') {
1737             $romfile = 'pxe-ne2k_pci.rom';
1738         } elsif ($device eq 'pcnet') {
1739             $romfile = 'pxe-pcnet.rom';
1740         } elsif ($device eq 'rtl8139') {
1741             $romfile = 'pxe-rtl8139.rom';
1742         }
1743         $tmpstr .= ",romfile=$romfile" if $romfile;
1744     }
1745
1746     return $tmpstr;
1747 }
1748
1749 sub print_netdev_full {
1750     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1751
1752     my $i = '';
1753     if ($netid =~ m/^net(\d+)$/) {
1754         $i = int($1);
1755     }
1756
1757     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1758
1759     my $ifname = "tap${vmid}i$i";
1760
1761     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1762     die "interface name '$ifname' is too long (max 15 character)\n"
1763         if length($ifname) >= 16;
1764
1765     my $vhostparam = '';
1766     if (is_native($arch)) {
1767         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1768     }
1769
1770     my $vmname = $conf->{name} || "vm$vmid";
1771
1772     my $netdev = "";
1773     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1774
1775     if ($net->{bridge}) {
1776         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1777             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1778     } else {
1779         $netdev = "type=user,id=$netid,hostname=$vmname";
1780     }
1781
1782     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1783
1784     return $netdev;
1785 }
1786
1787 my $vga_map = {
1788     'cirrus' => 'cirrus-vga',
1789     'std' => 'VGA',
1790     'vmware' => 'vmware-svga',
1791     'virtio' => 'virtio-vga',
1792     'virtio-gl' => 'virtio-vga-gl',
1793 };
1794
1795 sub print_vga_device {
1796     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1797
1798     my $type = $vga_map->{$vga->{type}};
1799     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1800         $type = 'virtio-gpu';
1801     }
1802     my $vgamem_mb = $vga->{memory};
1803
1804     my $max_outputs = '';
1805     if ($qxlnum) {
1806         $type = $id ? 'qxl' : 'qxl-vga';
1807
1808         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1809             # set max outputs so linux can have up to 4 qxl displays with one device
1810             if (min_version($machine_version, 4, 1)) {
1811                 $max_outputs = ",max_outputs=4";
1812             }
1813         }
1814     }
1815
1816     die "no devicetype for $vga->{type}\n" if !$type;
1817
1818     my $memory = "";
1819     if ($vgamem_mb) {
1820         if ($vga->{type} =~ /^virtio/) {
1821             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1822             $memory = ",max_hostmem=$bytes";
1823         } elsif ($qxlnum) {
1824             # from https://www.spice-space.org/multiple-monitors.html
1825             $memory = ",vgamem_mb=$vga->{memory}";
1826             my $ram = $vgamem_mb * 4;
1827             my $vram = $vgamem_mb * 2;
1828             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1829         } else {
1830             $memory = ",vgamem_mb=$vga->{memory}";
1831         }
1832     } elsif ($qxlnum && $id) {
1833         $memory = ",ram_size=67108864,vram_size=33554432";
1834     }
1835
1836     my $edidoff = "";
1837     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1838         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1839     }
1840
1841     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1842     my $vgaid = "vga" . ($id // '');
1843     my $pciaddr;
1844     if ($q35 && $vgaid eq 'vga') {
1845         # the first display uses pcie.0 bus on q35 machines
1846         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1847     } else {
1848         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1849     }
1850
1851     if ($vga->{type} eq 'virtio-gl') {
1852         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1853         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1854             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1855
1856         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1857             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1858     }
1859
1860     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1861 }
1862
1863 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1864 sub parse_net {
1865     my ($data, $disable_mac_autogen) = @_;
1866
1867     my $res = eval { parse_property_string($net_fmt, $data) };
1868     if ($@) {
1869         warn $@;
1870         return;
1871     }
1872     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1873         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1874         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1875     }
1876     return $res;
1877 }
1878
1879 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1880 sub parse_ipconfig {
1881     my ($data) = @_;
1882
1883     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1884     if ($@) {
1885         warn $@;
1886         return;
1887     }
1888
1889     if ($res->{gw} && !$res->{ip}) {
1890         warn 'gateway specified without specifying an IP address';
1891         return;
1892     }
1893     if ($res->{gw6} && !$res->{ip6}) {
1894         warn 'IPv6 gateway specified without specifying an IPv6 address';
1895         return;
1896     }
1897     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1898         warn 'gateway specified together with DHCP';
1899         return;
1900     }
1901     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1902         # gw6 + auto/dhcp
1903         warn "IPv6 gateway specified together with $res->{ip6} address";
1904         return;
1905     }
1906
1907     if (!$res->{ip} && !$res->{ip6}) {
1908         return { ip => 'dhcp', ip6 => 'dhcp' };
1909     }
1910
1911     return $res;
1912 }
1913
1914 sub print_net {
1915     my $net = shift;
1916
1917     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1918 }
1919
1920 sub add_random_macs {
1921     my ($settings) = @_;
1922
1923     foreach my $opt (keys %$settings) {
1924         next if $opt !~ m/^net(\d+)$/;
1925         my $net = parse_net($settings->{$opt});
1926         next if !$net;
1927         $settings->{$opt} = print_net($net);
1928     }
1929 }
1930
1931 sub vm_is_volid_owner {
1932     my ($storecfg, $vmid, $volid) = @_;
1933
1934     if ($volid !~  m|^/|) {
1935         my ($path, $owner);
1936         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1937         if ($owner && ($owner == $vmid)) {
1938             return 1;
1939         }
1940     }
1941
1942     return;
1943 }
1944
1945 sub vmconfig_register_unused_drive {
1946     my ($storecfg, $vmid, $conf, $drive) = @_;
1947
1948     if (drive_is_cloudinit($drive)) {
1949         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1950         warn $@ if $@;
1951         delete $conf->{cloudinit};
1952     } elsif (!drive_is_cdrom($drive)) {
1953         my $volid = $drive->{file};
1954         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1955             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1956         }
1957     }
1958 }
1959
1960 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1961 my $smbios1_fmt = {
1962     uuid => {
1963         type => 'string',
1964         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1965         format_description => 'UUID',
1966         description => "Set SMBIOS1 UUID.",
1967         optional => 1,
1968     },
1969     version => {
1970         type => 'string',
1971         pattern => '[A-Za-z0-9+\/]+={0,2}',
1972         format_description => 'Base64 encoded string',
1973         description => "Set SMBIOS1 version.",
1974         optional => 1,
1975     },
1976     serial => {
1977         type => 'string',
1978         pattern => '[A-Za-z0-9+\/]+={0,2}',
1979         format_description => 'Base64 encoded string',
1980         description => "Set SMBIOS1 serial number.",
1981         optional => 1,
1982     },
1983     manufacturer => {
1984         type => 'string',
1985         pattern => '[A-Za-z0-9+\/]+={0,2}',
1986         format_description => 'Base64 encoded string',
1987         description => "Set SMBIOS1 manufacturer.",
1988         optional => 1,
1989     },
1990     product => {
1991         type => 'string',
1992         pattern => '[A-Za-z0-9+\/]+={0,2}',
1993         format_description => 'Base64 encoded string',
1994         description => "Set SMBIOS1 product ID.",
1995         optional => 1,
1996     },
1997     sku => {
1998         type => 'string',
1999         pattern => '[A-Za-z0-9+\/]+={0,2}',
2000         format_description => 'Base64 encoded string',
2001         description => "Set SMBIOS1 SKU string.",
2002         optional => 1,
2003     },
2004     family => {
2005         type => 'string',
2006         pattern => '[A-Za-z0-9+\/]+={0,2}',
2007         format_description => 'Base64 encoded string',
2008         description => "Set SMBIOS1 family string.",
2009         optional => 1,
2010     },
2011     base64 => {
2012         type => 'boolean',
2013         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2014         optional => 1,
2015     },
2016 };
2017
2018 sub parse_smbios1 {
2019     my ($data) = @_;
2020
2021     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2022     warn $@ if $@;
2023     return $res;
2024 }
2025
2026 sub print_smbios1 {
2027     my ($smbios1) = @_;
2028     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2029 }
2030
2031 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2032
2033 sub parse_watchdog {
2034     my ($value) = @_;
2035
2036     return if !$value;
2037
2038     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2039     warn $@ if $@;
2040     return $res;
2041 }
2042
2043 sub parse_guest_agent {
2044     my ($conf) = @_;
2045
2046     return {} if !defined($conf->{agent});
2047
2048     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2049     warn $@ if $@;
2050
2051     # if the agent is disabled ignore the other potentially set properties
2052     return {} if !$res->{enabled};
2053     return $res;
2054 }
2055
2056 sub get_qga_key {
2057     my ($conf, $key) = @_;
2058     return undef if !defined($conf->{agent});
2059
2060     my $agent = parse_guest_agent($conf);
2061     return $agent->{$key};
2062 }
2063
2064 sub parse_vga {
2065     my ($value) = @_;
2066
2067     return {} if !$value;
2068     my $res = eval { parse_property_string($vga_fmt, $value) };
2069     warn $@ if $@;
2070     return $res;
2071 }
2072
2073 sub parse_rng {
2074     my ($value) = @_;
2075
2076     return if !$value;
2077
2078     my $res = eval { parse_property_string($rng_fmt, $value) };
2079     warn $@ if $@;
2080     return $res;
2081 }
2082
2083 sub parse_meta_info {
2084     my ($value) = @_;
2085
2086     return if !$value;
2087
2088     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2089     warn $@ if $@;
2090     return $res;
2091 }
2092
2093 sub new_meta_info_string {
2094     my () = @_; # for now do not allow to override any value
2095
2096     return PVE::JSONSchema::print_property_string(
2097         {
2098             'creation-qemu' => kvm_user_version(),
2099             ctime => "". int(time()),
2100         },
2101         $meta_info_fmt
2102     );
2103 }
2104
2105 sub qemu_created_version_fixups {
2106     my ($conf, $forcemachine, $kvmver) = @_;
2107
2108     my $meta = parse_meta_info($conf->{meta}) // {};
2109     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2110
2111     # check if we need to apply some handling for VMs that always use the latest machine version but
2112     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2113     # would be required (we do not want to pin machine version for non-windows OS type)
2114     if (
2115         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2116         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2117         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2118         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2119     ) {
2120         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2121         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2122             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2123             # and thus with the predictable interface naming of systemd
2124             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2125         }
2126     }
2127     return;
2128 }
2129
2130 # add JSON properties for create and set function
2131 sub json_config_properties {
2132     my ($prop, $with_disk_alloc) = @_;
2133
2134     my $skip_json_config_opts = {
2135         parent => 1,
2136         snaptime => 1,
2137         vmstate => 1,
2138         runningmachine => 1,
2139         runningcpu => 1,
2140         meta => 1,
2141     };
2142
2143     foreach my $opt (keys %$confdesc) {
2144         next if $skip_json_config_opts->{$opt};
2145
2146         if ($with_disk_alloc && is_valid_drivename($opt)) {
2147             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2148         } else {
2149             $prop->{$opt} = $confdesc->{$opt};
2150         }
2151     }
2152
2153     return $prop;
2154 }
2155
2156 # Properties that we can read from an OVF file
2157 sub json_ovf_properties {
2158     my $prop = {};
2159
2160     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2161         $prop->{$device} = {
2162             type => 'string',
2163             format => 'pve-volume-id-or-absolute-path',
2164             description => "Disk image that gets imported to $device",
2165             optional => 1,
2166         };
2167     }
2168
2169     $prop->{cores} = {
2170         type => 'integer',
2171         description => "The number of CPU cores.",
2172         optional => 1,
2173     };
2174     $prop->{memory} = {
2175         type => 'integer',
2176         description => "Amount of RAM for the VM in MB.",
2177         optional => 1,
2178     };
2179     $prop->{name} = {
2180         type => 'string',
2181         description => "Name of the VM.",
2182         optional => 1,
2183     };
2184
2185     return $prop;
2186 }
2187
2188 # return copy of $confdesc_cloudinit to generate documentation
2189 sub cloudinit_config_properties {
2190
2191     return dclone($confdesc_cloudinit);
2192 }
2193
2194 sub cloudinit_pending_properties {
2195     my $p = {
2196         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2197         name => 1,
2198     };
2199     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2200     return $p;
2201 }
2202
2203 sub check_type {
2204     my ($key, $value) = @_;
2205
2206     die "unknown setting '$key'\n" if !$confdesc->{$key};
2207
2208     my $type = $confdesc->{$key}->{type};
2209
2210     if (!defined($value)) {
2211         die "got undefined value\n";
2212     }
2213
2214     if ($value =~ m/[\n\r]/) {
2215         die "property contains a line feed\n";
2216     }
2217
2218     if ($type eq 'boolean') {
2219         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2220         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2221         die "type check ('boolean') failed - got '$value'\n";
2222     } elsif ($type eq 'integer') {
2223         return int($1) if $value =~ m/^(\d+)$/;
2224         die "type check ('integer') failed - got '$value'\n";
2225     } elsif ($type eq 'number') {
2226         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2227         die "type check ('number') failed - got '$value'\n";
2228     } elsif ($type eq 'string') {
2229         if (my $fmt = $confdesc->{$key}->{format}) {
2230             PVE::JSONSchema::check_format($fmt, $value);
2231             return $value;
2232         }
2233         $value =~ s/^\"(.*)\"$/$1/;
2234         return $value;
2235     } else {
2236         die "internal error"
2237     }
2238 }
2239
2240 sub destroy_vm {
2241     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2242
2243     my $conf = PVE::QemuConfig->load_config($vmid);
2244
2245     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2246         PVE::QemuConfig->check_lock($conf);
2247     }
2248
2249     if ($conf->{template}) {
2250         # check if any base image is still used by a linked clone
2251         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2252                 my ($ds, $drive) = @_;
2253                 return if drive_is_cdrom($drive);
2254
2255                 my $volid = $drive->{file};
2256                 return if !$volid || $volid =~ m|^/|;
2257
2258                 die "base volume '$volid' is still in use by linked cloned\n"
2259                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2260
2261         });
2262     }
2263
2264     my $volids = {};
2265     my $remove_owned_drive = sub {
2266         my ($ds, $drive) = @_;
2267         return if drive_is_cdrom($drive, 1);
2268
2269         my $volid = $drive->{file};
2270         return if !$volid || $volid =~ m|^/|;
2271         return if $volids->{$volid};
2272
2273         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2274         return if !$path || !$owner || ($owner != $vmid);
2275
2276         $volids->{$volid} = 1;
2277         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2278         warn "Could not remove disk '$volid', check manually: $@" if $@;
2279     };
2280
2281     # only remove disks owned by this VM (referenced in the config)
2282     my $include_opts = {
2283         include_unused => 1,
2284         extra_keys => ['vmstate'],
2285     };
2286     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2287
2288     for my $snap (values %{$conf->{snapshots}}) {
2289         next if !defined($snap->{vmstate});
2290         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2291         next if !defined($drive);
2292         $remove_owned_drive->('vmstate', $drive);
2293     }
2294
2295     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2296
2297     if ($purge_unreferenced) { # also remove unreferenced disk
2298         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2299         PVE::Storage::foreach_volid($vmdisks, sub {
2300             my ($volid, $sid, $volname, $d) = @_;
2301             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2302             warn $@ if $@;
2303         });
2304     }
2305
2306     eval { delete_ifaces_ipams_ips($conf, $vmid)};
2307     warn $@ if $@;
2308
2309     if (defined $replacement_conf) {
2310         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2311     } else {
2312         PVE::QemuConfig->destroy_config($vmid);
2313     }
2314 }
2315
2316 sub parse_vm_config {
2317     my ($filename, $raw, $strict) = @_;
2318
2319     return if !defined($raw);
2320
2321     my $res = {
2322         digest => Digest::SHA::sha1_hex($raw),
2323         snapshots => {},
2324         pending => {},
2325         cloudinit => {},
2326     };
2327
2328     my $handle_error = sub {
2329         my ($msg) = @_;
2330
2331         if ($strict) {
2332             die $msg;
2333         } else {
2334             warn $msg;
2335         }
2336     };
2337
2338     $filename =~ m|/qemu-server/(\d+)\.conf$|
2339         || die "got strange filename '$filename'";
2340
2341     my $vmid = $1;
2342
2343     my $conf = $res;
2344     my $descr;
2345     my $finish_description = sub {
2346         if (defined($descr)) {
2347             $descr =~ s/\s+$//;
2348             $conf->{description} = $descr;
2349         }
2350         $descr = undef;
2351     };
2352     my $section = '';
2353
2354     my @lines = split(/\n/, $raw);
2355     foreach my $line (@lines) {
2356         next if $line =~ m/^\s*$/;
2357
2358         if ($line =~ m/^\[PENDING\]\s*$/i) {
2359             $section = 'pending';
2360             $finish_description->();
2361             $conf = $res->{$section} = {};
2362             next;
2363         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2364             $section = 'cloudinit';
2365             $finish_description->();
2366             $conf = $res->{$section} = {};
2367             next;
2368
2369         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2370             $section = $1;
2371             $finish_description->();
2372             $conf = $res->{snapshots}->{$section} = {};
2373             next;
2374         }
2375
2376         if ($line =~ m/^\#(.*)$/) {
2377             $descr = '' if !defined($descr);
2378             $descr .= PVE::Tools::decode_text($1) . "\n";
2379             next;
2380         }
2381
2382         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2383             $descr = '' if !defined($descr);
2384             $descr .= PVE::Tools::decode_text($2);
2385         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2386             $conf->{snapstate} = $1;
2387         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2388             my $key = $1;
2389             my $value = $2;
2390             $conf->{$key} = $value;
2391         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2392             my $value = $1;
2393             if ($section eq 'pending') {
2394                 $conf->{delete} = $value; # we parse this later
2395             } else {
2396                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2397             }
2398         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2399             my $key = $1;
2400             my $value = $2;
2401             if ($section eq 'cloudinit') {
2402                 # ignore validation only used for informative purpose
2403                 $conf->{$key} = $value;
2404                 next;
2405             }
2406             eval { $value = check_type($key, $value); };
2407             if ($@) {
2408                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2409             } else {
2410                 $key = 'ide2' if $key eq 'cdrom';
2411                 my $fmt = $confdesc->{$key}->{format};
2412                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2413                     my $v = parse_drive($key, $value);
2414                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2415                         $v->{file} = $volid;
2416                         $value = print_drive($v);
2417                     } else {
2418                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2419                         next;
2420                     }
2421                 }
2422
2423                 $conf->{$key} = $value;
2424             }
2425         } else {
2426             $handle_error->("vm $vmid - unable to parse config: $line\n");
2427         }
2428     }
2429
2430     $finish_description->();
2431     delete $res->{snapstate}; # just to be sure
2432
2433     return $res;
2434 }
2435
2436 sub write_vm_config {
2437     my ($filename, $conf) = @_;
2438
2439     delete $conf->{snapstate}; # just to be sure
2440
2441     if ($conf->{cdrom}) {
2442         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2443         $conf->{ide2} = $conf->{cdrom};
2444         delete $conf->{cdrom};
2445     }
2446
2447     # we do not use 'smp' any longer
2448     if ($conf->{sockets}) {
2449         delete $conf->{smp};
2450     } elsif ($conf->{smp}) {
2451         $conf->{sockets} = $conf->{smp};
2452         delete $conf->{cores};
2453         delete $conf->{smp};
2454     }
2455
2456     my $used_volids = {};
2457
2458     my $cleanup_config = sub {
2459         my ($cref, $pending, $snapname) = @_;
2460
2461         foreach my $key (keys %$cref) {
2462             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2463                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2464             my $value = $cref->{$key};
2465             if ($key eq 'delete') {
2466                 die "propertry 'delete' is only allowed in [PENDING]\n"
2467                     if !$pending;
2468                 # fixme: check syntax?
2469                 next;
2470             }
2471             eval { $value = check_type($key, $value); };
2472             die "unable to parse value of '$key' - $@" if $@;
2473
2474             $cref->{$key} = $value;
2475
2476             if (!$snapname && is_valid_drivename($key)) {
2477                 my $drive = parse_drive($key, $value);
2478                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2479             }
2480         }
2481     };
2482
2483     &$cleanup_config($conf);
2484
2485     &$cleanup_config($conf->{pending}, 1);
2486
2487     foreach my $snapname (keys %{$conf->{snapshots}}) {
2488         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2489         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2490     }
2491
2492     # remove 'unusedX' settings if we re-add a volume
2493     foreach my $key (keys %$conf) {
2494         my $value = $conf->{$key};
2495         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2496             delete $conf->{$key};
2497         }
2498     }
2499
2500     my $generate_raw_config = sub {
2501         my ($conf, $pending) = @_;
2502
2503         my $raw = '';
2504
2505         # add description as comment to top of file
2506         if (defined(my $descr = $conf->{description})) {
2507             if ($descr) {
2508                 foreach my $cl (split(/\n/, $descr)) {
2509                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2510                 }
2511             } else {
2512                 $raw .= "#\n" if $pending;
2513             }
2514         }
2515
2516         foreach my $key (sort keys %$conf) {
2517             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2518             $raw .= "$key: $conf->{$key}\n";
2519         }
2520         return $raw;
2521     };
2522
2523     my $raw = &$generate_raw_config($conf);
2524
2525     if (scalar(keys %{$conf->{pending}})){
2526         $raw .= "\n[PENDING]\n";
2527         $raw .= &$generate_raw_config($conf->{pending}, 1);
2528     }
2529
2530     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2531         $raw .= "\n[special:cloudinit]\n";
2532         $raw .= &$generate_raw_config($conf->{cloudinit});
2533     }
2534
2535     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2536         $raw .= "\n[$snapname]\n";
2537         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2538     }
2539
2540     return $raw;
2541 }
2542
2543 sub load_defaults {
2544
2545     my $res = {};
2546
2547     # we use static defaults from our JSON schema configuration
2548     foreach my $key (keys %$confdesc) {
2549         if (defined(my $default = $confdesc->{$key}->{default})) {
2550             $res->{$key} = $default;
2551         }
2552     }
2553
2554     return $res;
2555 }
2556
2557 sub config_list {
2558     my $vmlist = PVE::Cluster::get_vmlist();
2559     my $res = {};
2560     return $res if !$vmlist || !$vmlist->{ids};
2561     my $ids = $vmlist->{ids};
2562     my $nodename = nodename();
2563
2564     foreach my $vmid (keys %$ids) {
2565         my $d = $ids->{$vmid};
2566         next if !$d->{node} || $d->{node} ne $nodename;
2567         next if !$d->{type} || $d->{type} ne 'qemu';
2568         $res->{$vmid}->{exists} = 1;
2569     }
2570     return $res;
2571 }
2572
2573 # test if VM uses local resources (to prevent migration)
2574 sub check_local_resources {
2575     my ($conf, $noerr) = @_;
2576
2577     my @loc_res = ();
2578     my $mapped_res = [];
2579
2580     my $nodelist = PVE::Cluster::get_nodelist();
2581     my $pci_map = PVE::Mapping::PCI::config();
2582     my $usb_map = PVE::Mapping::USB::config();
2583
2584     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2585
2586     my $add_missing_mapping = sub {
2587         my ($type, $key, $id) = @_;
2588         for my $node (@$nodelist) {
2589             my $entry;
2590             if ($type eq 'pci') {
2591                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2592             } elsif ($type eq 'usb') {
2593                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2594             }
2595             if (!scalar($entry->@*)) {
2596                 push @{$missing_mappings_by_node->{$node}}, $key;
2597             }
2598         }
2599     };
2600
2601     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2602     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2603
2604     push @loc_res, "ivshmem" if $conf->{ivshmem};
2605
2606     foreach my $k (keys %$conf) {
2607         if ($k =~ m/^usb/) {
2608             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2609             next if $entry->{host} =~ m/^spice$/i;
2610             if ($entry->{mapping}) {
2611                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2612                 push @$mapped_res, $k;
2613             }
2614         }
2615         if ($k =~ m/^hostpci/) {
2616             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2617             if ($entry->{mapping}) {
2618                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2619                 push @$mapped_res, $k;
2620             }
2621         }
2622         # sockets are safe: they will recreated be on the target side post-migrate
2623         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2624         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2625     }
2626
2627     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2628
2629     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2630 }
2631
2632 # check if used storages are available on all nodes (use by migrate)
2633 sub check_storage_availability {
2634     my ($storecfg, $conf, $node) = @_;
2635
2636     PVE::QemuConfig->foreach_volume($conf, sub {
2637         my ($ds, $drive) = @_;
2638
2639         my $volid = $drive->{file};
2640         return if !$volid;
2641
2642         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2643         return if !$sid;
2644
2645         # check if storage is available on both nodes
2646         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2647         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2648
2649         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2650
2651         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2652             if !$scfg->{content}->{$vtype};
2653    });
2654 }
2655
2656 # list nodes where all VM images are available (used by has_feature API)
2657 sub shared_nodes {
2658     my ($conf, $storecfg) = @_;
2659
2660     my $nodelist = PVE::Cluster::get_nodelist();
2661     my $nodehash = { map { $_ => 1 } @$nodelist };
2662     my $nodename = nodename();
2663
2664     PVE::QemuConfig->foreach_volume($conf, sub {
2665         my ($ds, $drive) = @_;
2666
2667         my $volid = $drive->{file};
2668         return if !$volid;
2669
2670         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2671         if ($storeid) {
2672             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2673             if ($scfg->{disable}) {
2674                 $nodehash = {};
2675             } elsif (my $avail = $scfg->{nodes}) {
2676                 foreach my $node (keys %$nodehash) {
2677                     delete $nodehash->{$node} if !$avail->{$node};
2678                 }
2679             } elsif (!$scfg->{shared}) {
2680                 foreach my $node (keys %$nodehash) {
2681                     delete $nodehash->{$node} if $node ne $nodename
2682                 }
2683             }
2684         }
2685     });
2686
2687     return $nodehash
2688 }
2689
2690 sub check_local_storage_availability {
2691     my ($conf, $storecfg) = @_;
2692
2693     my $nodelist = PVE::Cluster::get_nodelist();
2694     my $nodehash = { map { $_ => {} } @$nodelist };
2695
2696     PVE::QemuConfig->foreach_volume($conf, sub {
2697         my ($ds, $drive) = @_;
2698
2699         my $volid = $drive->{file};
2700         return if !$volid;
2701
2702         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2703         if ($storeid) {
2704             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2705
2706             if ($scfg->{disable}) {
2707                 foreach my $node (keys %$nodehash) {
2708                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2709                 }
2710             } elsif (my $avail = $scfg->{nodes}) {
2711                 foreach my $node (keys %$nodehash) {
2712                     if (!$avail->{$node}) {
2713                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2714                     }
2715                 }
2716             }
2717         }
2718     });
2719
2720     foreach my $node (values %$nodehash) {
2721         if (my $unavail = $node->{unavailable_storages}) {
2722             $node->{unavailable_storages} = [ sort keys %$unavail ];
2723         }
2724     }
2725
2726     return $nodehash
2727 }
2728
2729 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2730 sub check_running {
2731     my ($vmid, $nocheck, $node) = @_;
2732
2733     # $nocheck is set when called during a migration, in which case the config
2734     # file might still or already reside on the *other* node
2735     # - because rename has already happened, and current node is source
2736     # - because rename hasn't happened yet, and current node is target
2737     # - because rename has happened, current node is target, but hasn't yet
2738     # processed it yet
2739     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2740     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2741 }
2742
2743 sub vzlist {
2744
2745     my $vzlist = config_list();
2746
2747     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2748
2749     while (defined(my $de = $fd->read)) {
2750         next if $de !~ m/^(\d+)\.pid$/;
2751         my $vmid = $1;
2752         next if !defined($vzlist->{$vmid});
2753         if (my $pid = check_running($vmid)) {
2754             $vzlist->{$vmid}->{pid} = $pid;
2755         }
2756     }
2757
2758     return $vzlist;
2759 }
2760
2761 our $vmstatus_return_properties = {
2762     vmid => get_standard_option('pve-vmid'),
2763     status => {
2764         description => "QEMU process status.",
2765         type => 'string',
2766         enum => ['stopped', 'running'],
2767     },
2768     maxmem => {
2769         description => "Maximum memory in bytes.",
2770         type => 'integer',
2771         optional => 1,
2772         renderer => 'bytes',
2773     },
2774     maxdisk => {
2775         description => "Root disk size in bytes.",
2776         type => 'integer',
2777         optional => 1,
2778         renderer => 'bytes',
2779     },
2780     name => {
2781         description => "VM name.",
2782         type => 'string',
2783         optional => 1,
2784     },
2785     qmpstatus => {
2786         description => "VM run state from the 'query-status' QMP monitor command.",
2787         type => 'string',
2788         optional => 1,
2789     },
2790     pid => {
2791         description => "PID of running qemu process.",
2792         type => 'integer',
2793         optional => 1,
2794     },
2795     uptime => {
2796         description => "Uptime.",
2797         type => 'integer',
2798         optional => 1,
2799         renderer => 'duration',
2800     },
2801     cpus => {
2802         description => "Maximum usable CPUs.",
2803         type => 'number',
2804         optional => 1,
2805     },
2806     lock => {
2807         description => "The current config lock, if any.",
2808         type => 'string',
2809         optional => 1,
2810     },
2811     tags => {
2812         description  => "The current configured tags, if any",
2813         type => 'string',
2814         optional => 1,
2815     },
2816     'running-machine' => {
2817         description => "The currently running machine type (if running).",
2818         type => 'string',
2819         optional => 1,
2820     },
2821     'running-qemu' => {
2822         description => "The currently running QEMU version (if running).",
2823         type => 'string',
2824         optional => 1,
2825     },
2826 };
2827
2828 my $last_proc_pid_stat;
2829
2830 # get VM status information
2831 # This must be fast and should not block ($full == false)
2832 # We only query KVM using QMP if $full == true (this can be slow)
2833 sub vmstatus {
2834     my ($opt_vmid, $full) = @_;
2835
2836     my $res = {};
2837
2838     my $storecfg = PVE::Storage::config();
2839
2840     my $list = vzlist();
2841     my $defaults = load_defaults();
2842
2843     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2844
2845     my $cpucount = $cpuinfo->{cpus} || 1;
2846
2847     foreach my $vmid (keys %$list) {
2848         next if $opt_vmid && ($vmid ne $opt_vmid);
2849
2850         my $conf = PVE::QemuConfig->load_config($vmid);
2851
2852         my $d = { vmid => int($vmid) };
2853         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2854
2855         # fixme: better status?
2856         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2857
2858         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2859         if (defined($size)) {
2860             $d->{disk} = 0; # no info available
2861             $d->{maxdisk} = $size;
2862         } else {
2863             $d->{disk} = 0;
2864             $d->{maxdisk} = 0;
2865         }
2866
2867         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2868             * ($conf->{cores} || $defaults->{cores});
2869         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2870         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2871
2872         $d->{name} = $conf->{name} || "VM $vmid";
2873         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2874
2875         if ($conf->{balloon}) {
2876             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2877             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2878                 : $defaults->{shares};
2879         }
2880
2881         $d->{uptime} = 0;
2882         $d->{cpu} = 0;
2883         $d->{mem} = 0;
2884
2885         $d->{netout} = 0;
2886         $d->{netin} = 0;
2887
2888         $d->{diskread} = 0;
2889         $d->{diskwrite} = 0;
2890
2891         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2892
2893         $d->{serial} = 1 if conf_has_serial($conf);
2894         $d->{lock} = $conf->{lock} if $conf->{lock};
2895         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2896
2897         $res->{$vmid} = $d;
2898     }
2899
2900     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2901     foreach my $dev (keys %$netdev) {
2902         next if $dev !~ m/^tap([1-9]\d*)i/;
2903         my $vmid = $1;
2904         my $d = $res->{$vmid};
2905         next if !$d;
2906
2907         $d->{netout} += $netdev->{$dev}->{receive};
2908         $d->{netin} += $netdev->{$dev}->{transmit};
2909
2910         if ($full) {
2911             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2912             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2913         }
2914
2915     }
2916
2917     my $ctime = gettimeofday;
2918
2919     foreach my $vmid (keys %$list) {
2920
2921         my $d = $res->{$vmid};
2922         my $pid = $d->{pid};
2923         next if !$pid;
2924
2925         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2926         next if !$pstat; # not running
2927
2928         my $used = $pstat->{utime} + $pstat->{stime};
2929
2930         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2931
2932         if ($pstat->{vsize}) {
2933             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2934         }
2935
2936         my $old = $last_proc_pid_stat->{$pid};
2937         if (!$old) {
2938             $last_proc_pid_stat->{$pid} = {
2939                 time => $ctime,
2940                 used => $used,
2941                 cpu => 0,
2942             };
2943             next;
2944         }
2945
2946         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2947
2948         if ($dtime > 1000) {
2949             my $dutime = $used -  $old->{used};
2950
2951             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2952             $last_proc_pid_stat->{$pid} = {
2953                 time => $ctime,
2954                 used => $used,
2955                 cpu => $d->{cpu},
2956             };
2957         } else {
2958             $d->{cpu} = $old->{cpu};
2959         }
2960     }
2961
2962     return $res if !$full;
2963
2964     my $qmpclient = PVE::QMPClient->new();
2965
2966     my $ballooncb = sub {
2967         my ($vmid, $resp) = @_;
2968
2969         my $info = $resp->{'return'};
2970         return if !$info->{max_mem};
2971
2972         my $d = $res->{$vmid};
2973
2974         # use memory assigned to VM
2975         $d->{maxmem} = $info->{max_mem};
2976         $d->{balloon} = $info->{actual};
2977
2978         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2979             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2980             $d->{freemem} = $info->{free_mem};
2981         }
2982
2983         $d->{ballooninfo} = $info;
2984     };
2985
2986     my $blockstatscb = sub {
2987         my ($vmid, $resp) = @_;
2988         my $data = $resp->{'return'} || [];
2989         my $totalrdbytes = 0;
2990         my $totalwrbytes = 0;
2991
2992         for my $blockstat (@$data) {
2993             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2994             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2995
2996             $blockstat->{device} =~ s/drive-//;
2997             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2998         }
2999         $res->{$vmid}->{diskread} = $totalrdbytes;
3000         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3001     };
3002
3003     my $machinecb = sub {
3004         my ($vmid, $resp) = @_;
3005         my $data = $resp->{'return'} || [];
3006
3007         $res->{$vmid}->{'running-machine'} =
3008             PVE::QemuServer::Machine::current_from_query_machines($data);
3009     };
3010
3011     my $versioncb = sub {
3012         my ($vmid, $resp) = @_;
3013         my $data = $resp->{'return'} // {};
3014         my $version = 'unknown';
3015
3016         if (my $v = $data->{qemu}) {
3017             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3018         }
3019
3020         $res->{$vmid}->{'running-qemu'} = $version;
3021     };
3022
3023     my $statuscb = sub {
3024         my ($vmid, $resp) = @_;
3025
3026         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3027         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3028         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3029         # this fails if ballon driver is not loaded, so this must be
3030         # the last commnand (following command are aborted if this fails).
3031         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3032
3033         my $status = 'unknown';
3034         if (!defined($status = $resp->{'return'}->{status})) {
3035             warn "unable to get VM status\n";
3036             return;
3037         }
3038
3039         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3040     };
3041
3042     foreach my $vmid (keys %$list) {
3043         next if $opt_vmid && ($vmid ne $opt_vmid);
3044         next if !$res->{$vmid}->{pid}; # not running
3045         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3046     }
3047
3048     $qmpclient->queue_execute(undef, 2);
3049
3050     foreach my $vmid (keys %$list) {
3051         next if $opt_vmid && ($vmid ne $opt_vmid);
3052         next if !$res->{$vmid}->{pid}; #not running
3053
3054         # we can't use the $qmpclient since it might have already aborted on
3055         # 'query-balloon', but this might also fail for older versions...
3056         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3057         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3058     }
3059
3060     foreach my $vmid (keys %$list) {
3061         next if $opt_vmid && ($vmid ne $opt_vmid);
3062         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3063     }
3064
3065     return $res;
3066 }
3067
3068 sub conf_has_serial {
3069     my ($conf) = @_;
3070
3071     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3072         if ($conf->{"serial$i"}) {
3073             return 1;
3074         }
3075     }
3076
3077     return 0;
3078 }
3079
3080 sub conf_has_audio {
3081     my ($conf, $id) = @_;
3082
3083     $id //= 0;
3084     my $audio = $conf->{"audio$id"};
3085     return if !defined($audio);
3086
3087     my $audioproperties = parse_property_string($audio_fmt, $audio);
3088     my $audiodriver = $audioproperties->{driver} // 'spice';
3089
3090     return {
3091         dev => $audioproperties->{device},
3092         dev_id => "audiodev$id",
3093         backend => $audiodriver,
3094         backend_id => "$audiodriver-backend${id}",
3095     };
3096 }
3097
3098 sub audio_devs {
3099     my ($audio, $audiopciaddr, $machine_version) = @_;
3100
3101     my $devs = [];
3102
3103     my $id = $audio->{dev_id};
3104     my $audiodev = "";
3105     if (min_version($machine_version, 4, 2)) {
3106         $audiodev = ",audiodev=$audio->{backend_id}";
3107     }
3108
3109     if ($audio->{dev} eq 'AC97') {
3110         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3111     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3112         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3113         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3114         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3115     } else {
3116         die "unkown audio device '$audio->{dev}', implement me!";
3117     }
3118
3119     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3120
3121     return $devs;
3122 }
3123
3124 sub get_tpm_paths {
3125     my ($vmid) = @_;
3126     return {
3127         socket => "/var/run/qemu-server/$vmid.swtpm",
3128         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3129     };
3130 }
3131
3132 sub add_tpm_device {
3133     my ($vmid, $devices, $conf) = @_;
3134
3135     return if !$conf->{tpmstate0};
3136
3137     my $paths = get_tpm_paths($vmid);
3138
3139     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3140     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3141     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3142 }
3143
3144 sub start_swtpm {
3145     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3146
3147     return if !$tpmdrive;
3148
3149     my $state;
3150     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3151     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3152     if ($storeid) {
3153         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3154     } else {
3155         $state = $tpm->{file};
3156     }
3157
3158     my $paths = get_tpm_paths($vmid);
3159
3160     # during migration, we will get state from remote
3161     #
3162     if (!$migration) {
3163         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3164         my $setup_cmd = [
3165             "swtpm_setup",
3166             "--tpmstate",
3167             "file://$state",
3168             "--createek",
3169             "--create-ek-cert",
3170             "--create-platform-cert",
3171             "--lock-nvram",
3172             "--config",
3173             "/etc/swtpm_setup.conf", # do not use XDG configs
3174             "--runas",
3175             "0", # force creation as root, error if not possible
3176             "--not-overwrite", # ignore existing state, do not modify
3177         ];
3178
3179         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3180         # TPM 2.0 supports ECC crypto, use if possible
3181         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3182
3183         run_command($setup_cmd, outfunc => sub {
3184             print "swtpm_setup: $1\n";
3185         });
3186     }
3187
3188     # Used to distinguish different invocations in the log.
3189     my $log_prefix = "[id=" . int(time()) . "] ";
3190
3191     my $emulator_cmd = [
3192         "swtpm",
3193         "socket",
3194         "--tpmstate",
3195         "backend-uri=file://$state,mode=0600",
3196         "--ctrl",
3197         "type=unixio,path=$paths->{socket},mode=0600",
3198         "--pid",
3199         "file=$paths->{pid}",
3200         "--terminate", # terminate on QEMU disconnect
3201         "--daemon",
3202         "--log",
3203         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3204     ];
3205     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3206     run_command($emulator_cmd, outfunc => sub { print $1; });
3207
3208     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3209     while (! -e $paths->{pid}) {
3210         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3211         usleep(50_000);
3212     }
3213
3214     # return untainted PID of swtpm daemon so it can be killed on error
3215     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3216     return $1;
3217 }
3218
3219 sub vga_conf_has_spice {
3220     my ($vga) = @_;
3221
3222     my $vgaconf = parse_vga($vga);
3223     my $vgatype = $vgaconf->{type};
3224     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3225
3226     return $1 || 1;
3227 }
3228
3229 sub is_native($) {
3230     my ($arch) = @_;
3231     return get_host_arch() eq $arch;
3232 }
3233
3234 sub get_vm_arch {
3235     my ($conf) = @_;
3236     return $conf->{arch} // get_host_arch();
3237 }
3238
3239 my $default_machines = {
3240     x86_64 => 'pc',
3241     aarch64 => 'virt',
3242 };
3243
3244 sub get_installed_machine_version {
3245     my ($kvmversion) = @_;
3246     $kvmversion = kvm_user_version() if !defined($kvmversion);
3247     $kvmversion =~ m/^(\d+\.\d+)/;
3248     return $1;
3249 }
3250
3251 sub windows_get_pinned_machine_version {
3252     my ($machine, $base_version, $kvmversion) = @_;
3253
3254     my $pin_version = $base_version;
3255     if (!defined($base_version) ||
3256         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3257     ) {
3258         $pin_version = get_installed_machine_version($kvmversion);
3259     }
3260     if (!$machine || $machine eq 'pc') {
3261         $machine = "pc-i440fx-$pin_version";
3262     } elsif ($machine eq 'q35') {
3263         $machine = "pc-q35-$pin_version";
3264     } elsif ($machine eq 'virt') {
3265         $machine = "virt-$pin_version";
3266     } else {
3267         warn "unknown machine type '$machine', not touching that!\n";
3268     }
3269
3270     return $machine;
3271 }
3272
3273 sub get_vm_machine {
3274     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3275
3276     my $machine = $forcemachine || $conf->{machine};
3277
3278     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3279         $kvmversion //= kvm_user_version();
3280         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3281         # layout which confuses windows quite a bit and may result in various regressions..
3282         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3283         if (windows_version($conf->{ostype})) {
3284             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3285         }
3286         $arch //= 'x86_64';
3287         $machine ||= $default_machines->{$arch};
3288         if ($add_pve_version) {
3289             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3290             $machine .= "+pve$pvever";
3291         }
3292     }
3293
3294     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3295         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3296         $machine = $1 if $is_pxe;
3297
3298         # for version-pinned machines that do not include a pve-version (e.g.
3299         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3300         $machine .= '+pve0';
3301
3302         $machine .= '.pxe' if $is_pxe;
3303     }
3304
3305     return $machine;
3306 }
3307
3308 sub get_ovmf_files($$$) {
3309     my ($arch, $efidisk, $smm) = @_;
3310
3311     my $types = $OVMF->{$arch}
3312         or die "no OVMF images known for architecture '$arch'\n";
3313
3314     my $type = 'default';
3315     if ($arch eq 'x86_64') {
3316         if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3317             $type = $smm ? "4m" : "4m-no-smm";
3318             $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3319         } else {
3320             # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3321         }
3322     }
3323
3324     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3325     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3326     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3327
3328     return ($ovmf_code, $ovmf_vars);
3329 }
3330
3331 my $Arch2Qemu = {
3332     aarch64 => '/usr/bin/qemu-system-aarch64',
3333     x86_64 => '/usr/bin/qemu-system-x86_64',
3334 };
3335 sub get_command_for_arch($) {
3336     my ($arch) = @_;
3337     return '/usr/bin/kvm' if is_native($arch);
3338
3339     my $cmd = $Arch2Qemu->{$arch}
3340         or die "don't know how to emulate architecture '$arch'\n";
3341     return $cmd;
3342 }
3343
3344 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3345 # to use in a QEMU command line (-cpu element), first array_intersect the result
3346 # of query_supported_ with query_understood_. This is necessary because:
3347 #
3348 # a) query_understood_ returns flags the host cannot use and
3349 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3350 #    flags, but CPU settings - with most of them being flags. Those settings
3351 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3352 #
3353 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3354 # expensive. If you need the value returned from this, you can get it much
3355 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3356 # $accel being 'kvm' or 'tcg'.
3357 #
3358 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3359 # changes, automatically populating pmxcfs.
3360 #
3361 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3362 # since kvm and tcg machines support different flags
3363 #
3364 sub query_supported_cpu_flags {
3365     my ($arch) = @_;
3366
3367     $arch //= get_host_arch();
3368     my $default_machine = $default_machines->{$arch};
3369
3370     my $flags = {};
3371
3372     # FIXME: Once this is merged, the code below should work for ARM as well:
3373     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3374     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3375         $arch eq "aarch64";
3376
3377     my $kvm_supported = defined(kvm_version());
3378     my $qemu_cmd = get_command_for_arch($arch);
3379     my $fakevmid = -1;
3380     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3381
3382     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3383     my $query_supported_run_qemu = sub {
3384         my ($kvm) = @_;
3385
3386         my $flags = {};
3387         my $cmd = [
3388             $qemu_cmd,
3389             '-machine', $default_machine,
3390             '-display', 'none',
3391             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3392             '-mon', 'chardev=qmp,mode=control',
3393             '-pidfile', $pidfile,
3394             '-S', '-daemonize'
3395         ];
3396
3397         if (!$kvm) {
3398             push @$cmd, '-accel', 'tcg';
3399         }
3400
3401         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3402         die "QEMU flag querying VM exited with code " . $rc if $rc;
3403
3404         eval {
3405             my $cmd_result = mon_cmd(
3406                 $fakevmid,
3407                 'query-cpu-model-expansion',
3408                 type => 'full',
3409                 model => { name => 'host' }
3410             );
3411
3412             my $props = $cmd_result->{model}->{props};
3413             foreach my $prop (keys %$props) {
3414                 next if $props->{$prop} ne '1';
3415                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3416                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3417                 # We only keep those with underscores, to match /proc/cpuinfo
3418                 $prop =~ s/\.|-/_/g;
3419                 $flags->{$prop} = 1;
3420             }
3421         };
3422         my $err = $@;
3423
3424         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3425         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3426
3427         die $err if $err;
3428
3429         return [ sort keys %$flags ];
3430     };
3431
3432     # We need to query QEMU twice, since KVM and TCG have different supported flags
3433     PVE::QemuConfig->lock_config($fakevmid, sub {
3434         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3435         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3436
3437         if ($kvm_supported) {
3438             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3439             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3440         }
3441     });
3442
3443     return $flags;
3444 }
3445
3446 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3447 my $understood_cpu_flag_dir = "/usr/share/kvm";
3448 sub query_understood_cpu_flags {
3449     my $arch = get_host_arch();
3450     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3451
3452     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3453         if ! -e $filepath;
3454
3455     my $raw = file_get_contents($filepath);
3456     $raw =~ s/^\s+|\s+$//g;
3457     my @flags = split(/\s+/, $raw);
3458
3459     return \@flags;
3460 }
3461
3462 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3463 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3464 my sub should_disable_smm {
3465     my ($conf, $vga, $machine) = @_;
3466
3467     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3468
3469     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3470         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3471 }
3472
3473 my sub print_ovmf_drive_commandlines {
3474     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3475
3476     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3477
3478     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3479
3480     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3481     if ($d) {
3482         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3483         my ($path, $format) = $d->@{'file', 'format'};
3484         if ($storeid) {
3485             $path = PVE::Storage::path($storecfg, $d->{file});
3486             if (!defined($format)) {
3487                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3488                 $format = qemu_img_format($scfg, $volname);
3489             }
3490         } elsif (!defined($format)) {
3491             die "efidisk format must be specified\n";
3492         }
3493         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3494         if ($path =~ m/^rbd:/) {
3495             $var_drive_str .= ',cache=writeback';
3496             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3497         }
3498         $var_drive_str .= ",format=$format,file=$path";
3499
3500         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3501         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3502     } else {
3503         log_warn("no efidisk configured! Using temporary efivars disk.");
3504         my $path = "/tmp/$vmid-ovmf.fd";
3505         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3506         $var_drive_str .= ",format=raw,file=$path";
3507         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3508     }
3509
3510     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3511 }
3512
3513 sub config_to_command {
3514     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3515         $pbs_backing) = @_;
3516
3517     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3518     my $devices = [];
3519     my $bridges = {};
3520     my $ostype = $conf->{ostype};
3521     my $winversion = windows_version($ostype);
3522     my $kvm = $conf->{kvm};
3523     my $nodename = nodename();
3524
3525     my $arch = get_vm_arch($conf);
3526     my $kvm_binary = get_command_for_arch($arch);
3527     my $kvmver = kvm_user_version($kvm_binary);
3528
3529     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3530         $kvmver //= "undefined";
3531         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3532     }
3533
3534     my $add_pve_version = min_version($kvmver, 4, 1);
3535
3536     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3537     my $machine_version = extract_version($machine_type, $kvmver);
3538     $kvm //= 1 if is_native($arch);
3539
3540     $machine_version =~ m/(\d+)\.(\d+)/;
3541     my ($machine_major, $machine_minor) = ($1, $2);
3542
3543     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3544         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3545     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3546         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3547             ." please upgrade node '$nodename'\n"
3548     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3549         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3550         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3551             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3552             ." node '$nodename'\n";
3553     }
3554
3555     # if a specific +pve version is required for a feature, use $version_guard
3556     # instead of min_version to allow machines to be run with the minimum
3557     # required version
3558     my $required_pve_version = 0;
3559     my $version_guard = sub {
3560         my ($major, $minor, $pve) = @_;
3561         return 0 if !min_version($machine_version, $major, $minor, $pve);
3562         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3563         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3564         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3565         return 1;
3566     };
3567
3568     if ($kvm && !defined kvm_version()) {
3569         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3570             ." or enable in BIOS.\n";
3571     }
3572
3573     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3574     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3575     my $use_old_bios_files = undef;
3576     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3577
3578     my $cmd = [];
3579     if ($conf->{affinity}) {
3580         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3581     }
3582
3583     push @$cmd, $kvm_binary;
3584
3585     push @$cmd, '-id', $vmid;
3586
3587     my $vmname = $conf->{name} || "vm$vmid";
3588
3589     push @$cmd, '-name', "$vmname,debug-threads=on";
3590
3591     push @$cmd, '-no-shutdown';
3592
3593     my $use_virtio = 0;
3594
3595     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3596     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3597     push @$cmd, '-mon', "chardev=qmp,mode=control";
3598
3599     if (min_version($machine_version, 2, 12)) {
3600         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3601         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3602     }
3603
3604     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3605
3606     push @$cmd, '-daemonize';
3607
3608     if ($conf->{smbios1}) {
3609         my $smbios_conf = parse_smbios1($conf->{smbios1});
3610         if ($smbios_conf->{base64}) {
3611             # Do not pass base64 flag to qemu
3612             delete $smbios_conf->{base64};
3613             my $smbios_string = "";
3614             foreach my $key (keys %$smbios_conf) {
3615                 my $value;
3616                 if ($key eq "uuid") {
3617                     $value = $smbios_conf->{uuid}
3618                 } else {
3619                     $value = decode_base64($smbios_conf->{$key});
3620                 }
3621                 # qemu accepts any binary data, only commas need escaping by double comma
3622                 $value =~ s/,/,,/g;
3623                 $smbios_string .= "," . $key . "=" . $value if $value;
3624             }
3625             push @$cmd, '-smbios', "type=1" . $smbios_string;
3626         } else {
3627             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3628         }
3629     }
3630
3631     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3632         my ($code_drive_str, $var_drive_str) =
3633             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3634         push $cmd->@*, '-drive', $code_drive_str;
3635         push $cmd->@*, '-drive', $var_drive_str;
3636     }
3637
3638     if ($q35) { # tell QEMU to load q35 config early
3639         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3640         if (min_version($machine_version, 4, 0)) {
3641             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3642         } else {
3643             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3644         }
3645     }
3646
3647     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3648         push @$cmd, $fixups->@*;
3649     }
3650
3651     if ($conf->{vmgenid}) {
3652         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3653     }
3654
3655     # add usb controllers
3656     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3657         $conf, $bridges, $arch, $machine_type, $machine_version);
3658     push @$devices, @usbcontrollers if @usbcontrollers;
3659     my $vga = parse_vga($conf->{vga});
3660
3661     my $qxlnum = vga_conf_has_spice($conf->{vga});
3662     $vga->{type} = 'qxl' if $qxlnum;
3663
3664     if (!$vga->{type}) {
3665         if ($arch eq 'aarch64') {
3666             $vga->{type} = 'virtio';
3667         } elsif (min_version($machine_version, 2, 9)) {
3668             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3669         } else {
3670             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3671         }
3672     }
3673
3674     # enable absolute mouse coordinates (needed by vnc)
3675     my $tablet = $conf->{tablet};
3676     if (!defined($tablet)) {
3677         $tablet = $defaults->{tablet};
3678         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3679         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3680     }
3681
3682     if ($tablet) {
3683         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3684         my $kbd = print_keyboarddevice_full($conf, $arch);
3685         push @$devices, '-device', $kbd if defined($kbd);
3686     }
3687
3688     my $bootorder = device_bootorder($conf);
3689
3690     # host pci device passthrough
3691     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3692         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3693
3694     # usb devices
3695     my $usb_dev_features = {};
3696     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3697
3698     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3699         $conf, $usb_dev_features, $bootorder, $machine_version);
3700     push @$devices, @usbdevices if @usbdevices;
3701
3702     # serial devices
3703     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3704         my $path = $conf->{"serial$i"} or next;
3705         if ($path eq 'socket') {
3706             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3707             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3708             # On aarch64, serial0 is the UART device. QEMU only allows
3709             # connecting UART devices via the '-serial' command line, as
3710             # the device has a fixed slot on the hardware...
3711             if ($arch eq 'aarch64' && $i == 0) {
3712                 push @$devices, '-serial', "chardev:serial$i";
3713             } else {
3714                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3715             }
3716         } else {
3717             die "no such serial device\n" if ! -c $path;
3718             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3719             push @$devices, '-device', "isa-serial,chardev=serial$i";
3720         }
3721     }
3722
3723     # parallel devices
3724     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3725         if (my $path = $conf->{"parallel$i"}) {
3726             die "no such parallel device\n" if ! -c $path;
3727             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3728             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3729             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3730         }
3731     }
3732
3733     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3734         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3735         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3736         push @$devices, @$audio_devs;
3737     }
3738
3739     # Add a TPM only if the VM is not a template,
3740     # to support backing up template VMs even if the TPM disk is write-protected.
3741     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3742
3743     my $sockets = 1;
3744     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3745     $sockets = $conf->{sockets} if  $conf->{sockets};
3746
3747     my $cores = $conf->{cores} || 1;
3748
3749     my $maxcpus = $sockets * $cores;
3750
3751     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3752
3753     my $allowed_vcpus = $cpuinfo->{cpus};
3754
3755     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3756
3757     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3758         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3759         for (my $i = 2; $i <= $vcpus; $i++)  {
3760             my $cpustr = print_cpu_device($conf,$i);
3761             push @$cmd, '-device', $cpustr;
3762         }
3763
3764     } else {
3765
3766         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3767     }
3768     push @$cmd, '-nodefaults';
3769
3770     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3771
3772     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3773
3774     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3775
3776     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3777         push @$devices, '-device', print_vga_device(
3778             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3779
3780         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3781
3782         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3783         push @$cmd,  '-vnc', "unix:$socket,password=on";
3784     } else {
3785         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3786         push @$cmd, '-nographic';
3787     }
3788
3789     # time drift fix
3790     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3791     my $useLocaltime = $conf->{localtime};
3792
3793     if ($winversion >= 5) { # windows
3794         $useLocaltime = 1 if !defined($conf->{localtime});
3795
3796         # use time drift fix when acpi is enabled
3797         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3798             $tdf = 1 if !defined($conf->{tdf});
3799         }
3800     }
3801
3802     if ($winversion >= 6) {
3803         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3804         push @$machineFlags, 'hpet=off';
3805     }
3806
3807     push @$rtcFlags, 'driftfix=slew' if $tdf;
3808
3809     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3810         push @$rtcFlags, "base=$conf->{startdate}";
3811     } elsif ($useLocaltime) {
3812         push @$rtcFlags, 'base=localtime';
3813     }
3814
3815     if ($forcecpu) {
3816         push @$cmd, '-cpu', $forcecpu;
3817     } else {
3818         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3819     }
3820
3821     PVE::QemuServer::Memory::config(
3822         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3823
3824     push @$cmd, '-S' if $conf->{freeze};
3825
3826     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3827
3828     my $guest_agent = parse_guest_agent($conf);
3829
3830     if ($guest_agent->{enabled}) {
3831         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3832         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3833
3834         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3835             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3836             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3837             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3838         } elsif ($guest_agent->{type} eq 'isa') {
3839             push @$devices, '-device', "isa-serial,chardev=qga0";
3840         }
3841     }
3842
3843     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3844     if ($rng && $version_guard->(4, 1, 2)) {
3845         check_rng_source($rng->{source});
3846
3847         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3848         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3849         my $limiter_str = "";
3850         if ($max_bytes) {
3851             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3852         }
3853
3854         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3855         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3856         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3857     }
3858
3859     my $spice_port;
3860
3861     assert_clipboard_config($vga);
3862     my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3863
3864     if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3865         if ($qxlnum > 1) {
3866             if ($winversion){
3867                 for (my $i = 1; $i < $qxlnum; $i++){
3868                     push @$devices, '-device', print_vga_device(
3869                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3870                 }
3871             } else {
3872                 # assume other OS works like Linux
3873                 my ($ram, $vram) = ("134217728", "67108864");
3874                 if ($vga->{memory}) {
3875                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3876                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3877                 }
3878                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3879                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3880             }
3881         }
3882
3883         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3884
3885         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3886         if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3887             push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3888         } else {
3889             push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3890         }
3891         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3892
3893         if ($is_spice) {
3894             my $pfamily = PVE::Tools::get_host_address_family($nodename);
3895             my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3896             die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3897
3898             my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3899             $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3900
3901             my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3902             my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3903             if ($spice_enhancement->{foldersharing}) {
3904                 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3905                 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3906             }
3907
3908             my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3909             $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3910                 if $spice_enhancement->{videostreaming};
3911             push @$devices, '-spice', "$spice_opts";
3912         }
3913     }
3914
3915     # enable balloon by default, unless explicitly disabled
3916     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3917         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3918         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3919         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3920         push @$devices, '-device', $ballooncmd;
3921     }
3922
3923     if ($conf->{watchdog}) {
3924         my $wdopts = parse_watchdog($conf->{watchdog});
3925         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3926         my $watchdog = $wdopts->{model} || 'i6300esb';
3927         push @$devices, '-device', "$watchdog$pciaddr";
3928         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3929     }
3930
3931     my $vollist = [];
3932     my $scsicontroller = {};
3933     my $ahcicontroller = {};
3934     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3935
3936     # Add iscsi initiator name if available
3937     if (my $initiator = get_initiator_name()) {
3938         push @$devices, '-iscsi', "initiator-name=$initiator";
3939     }
3940
3941     PVE::QemuConfig->foreach_volume($conf, sub {
3942         my ($ds, $drive) = @_;
3943
3944         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3945             check_volume_storage_type($storecfg, $drive->{file});
3946             push @$vollist, $drive->{file};
3947         }
3948
3949         # ignore efidisk here, already added in bios/fw handling code above
3950         return if $drive->{interface} eq 'efidisk';
3951         # similar for TPM
3952         return if $drive->{interface} eq 'tpmstate';
3953
3954         $use_virtio = 1 if $ds =~ m/^virtio/;
3955
3956         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3957
3958         if ($drive->{interface} eq 'virtio'){
3959            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3960         }
3961
3962         if ($drive->{interface} eq 'scsi') {
3963
3964             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3965
3966             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3967                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3968
3969             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3970             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3971
3972             my $iothread = '';
3973             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3974                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3975                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3976             } elsif ($drive->{iothread}) {
3977                 log_warn(
3978                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3979                 );
3980             }
3981
3982             my $queues = '';
3983             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3984                 $queues = ",num_queues=$drive->{queues}";
3985             }
3986
3987             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3988                 if !$scsicontroller->{$controller};
3989             $scsicontroller->{$controller}=1;
3990         }
3991
3992         if ($drive->{interface} eq 'sata') {
3993             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3994             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3995             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3996                 if !$ahcicontroller->{$controller};
3997             $ahcicontroller->{$controller}=1;
3998         }
3999
4000         my $pbs_conf = $pbs_backing->{$ds};
4001         my $pbs_name = undef;
4002         if ($pbs_conf) {
4003             $pbs_name = "drive-$ds-pbs";
4004             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4005         }
4006
4007         my $drive_cmd = print_drive_commandline_full(
4008             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4009
4010         # extra protection for templates, but SATA and IDE don't support it..
4011         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4012
4013         push @$devices, '-drive',$drive_cmd;
4014         push @$devices, '-device', print_drivedevice_full(
4015             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4016     });
4017
4018     for (my $i = 0; $i < $MAX_NETS; $i++) {
4019         my $netname = "net$i";
4020
4021         next if !$conf->{$netname};
4022         my $d = parse_net($conf->{$netname});
4023         next if !$d;
4024         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4025
4026         $use_virtio = 1 if $d->{model} eq 'virtio';
4027
4028         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4029
4030         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4031         push @$devices, '-netdev', $netdevfull;
4032
4033         my $netdevicefull = print_netdevice_full(
4034             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4035
4036         push @$devices, '-device', $netdevicefull;
4037     }
4038
4039     if ($conf->{ivshmem}) {
4040         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4041
4042         my $bus;
4043         if ($q35) {
4044             $bus = print_pcie_addr("ivshmem");
4045         } else {
4046             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4047         }
4048
4049         my $ivshmem_name = $ivshmem->{name} // $vmid;
4050         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4051
4052         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4053         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4054             .",size=$ivshmem->{size}M";
4055     }
4056
4057     # pci.4 is nested in pci.1
4058     $bridges->{1} = 1 if $bridges->{4};
4059
4060     if (!$q35) { # add pci bridges
4061         if (min_version($machine_version, 2, 3)) {
4062            $bridges->{1} = 1;
4063            $bridges->{2} = 1;
4064         }
4065         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4066     }
4067
4068     for my $k (sort {$b cmp $a} keys %$bridges) {
4069         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4070
4071         my $k_name = $k;
4072         if ($k == 2 && $legacy_igd) {
4073             $k_name = "$k-igd";
4074         }
4075         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4076         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4077
4078         if ($q35) { # add after -readconfig pve-q35.cfg
4079             splice @$devices, 2, 0, '-device', $devstr;
4080         } else {
4081             unshift @$devices, '-device', $devstr if $k > 0;
4082         }
4083     }
4084
4085     if (!$kvm) {
4086         push @$machineFlags, 'accel=tcg';
4087     }
4088
4089     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4090
4091     my $machine_type_min = $machine_type;
4092     if ($add_pve_version) {
4093         $machine_type_min =~ s/\+pve\d+$//;
4094         $machine_type_min .= "+pve$required_pve_version";
4095     }
4096     push @$machineFlags, "type=${machine_type_min}";
4097
4098     push @$cmd, @$devices;
4099     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4100     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4101     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4102
4103     if (my $vmstate = $conf->{vmstate}) {
4104         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4105         push @$vollist, $vmstate;
4106         push @$cmd, '-loadstate', $statepath;
4107         print "activating and using '$vmstate' as vmstate\n";
4108     }
4109
4110     if (PVE::QemuConfig->is_template($conf)) {
4111         # needed to workaround base volumes being read-only
4112         push @$cmd, '-snapshot';
4113     }
4114
4115     # add custom args
4116     if ($conf->{args}) {
4117         my $aa = PVE::Tools::split_args($conf->{args});
4118         push @$cmd, @$aa;
4119     }
4120
4121     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4122 }
4123
4124 sub check_rng_source {
4125     my ($source) = @_;
4126
4127     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4128     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4129         if ! -e $source;
4130
4131     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4132     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4133         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4134         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4135         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4136             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4137             ." to the host.\n";
4138     }
4139 }
4140
4141 sub spice_port {
4142     my ($vmid) = @_;
4143
4144     my $res = mon_cmd($vmid, 'query-spice');
4145
4146     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4147 }
4148
4149 sub vm_devices_list {
4150     my ($vmid) = @_;
4151
4152     my $res = mon_cmd($vmid, 'query-pci');
4153     my $devices_to_check = [];
4154     my $devices = {};
4155     foreach my $pcibus (@$res) {
4156         push @$devices_to_check, @{$pcibus->{devices}},
4157     }
4158
4159     while (@$devices_to_check) {
4160         my $to_check = [];
4161         for my $d (@$devices_to_check) {
4162             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4163             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4164
4165             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4166             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4167         }
4168         $devices_to_check = $to_check;
4169     }
4170
4171     my $resblock = mon_cmd($vmid, 'query-block');
4172     foreach my $block (@$resblock) {
4173         if($block->{device} =~ m/^drive-(\S+)/){
4174                 $devices->{$1} = 1;
4175         }
4176     }
4177
4178     my $resmice = mon_cmd($vmid, 'query-mice');
4179     foreach my $mice (@$resmice) {
4180         if ($mice->{name} eq 'QEMU HID Tablet') {
4181             $devices->{tablet} = 1;
4182             last;
4183         }
4184     }
4185
4186     # for usb devices there is no query-usb
4187     # but we can iterate over the entries in
4188     # qom-list path=/machine/peripheral
4189     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4190     foreach my $per (@$resperipheral) {
4191         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4192             $devices->{$per->{name}} = 1;
4193         }
4194     }
4195
4196     return $devices;
4197 }
4198
4199 sub vm_deviceplug {
4200     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4201
4202     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4203
4204     my $devices_list = vm_devices_list($vmid);
4205     return 1 if defined($devices_list->{$deviceid});
4206
4207     # add PCI bridge if we need it for the device
4208     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4209
4210     if ($deviceid eq 'tablet') {
4211         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4212     } elsif ($deviceid eq 'keyboard') {
4213         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4214     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4215         my $id = $1;
4216         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4217         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4218     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4219         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4220     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4221         qemu_iothread_add($vmid, $deviceid, $device);
4222
4223         qemu_driveadd($storecfg, $vmid, $device);
4224         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4225
4226         qemu_deviceadd($vmid, $devicefull);
4227         eval { qemu_deviceaddverify($vmid, $deviceid); };
4228         if (my $err = $@) {
4229             eval { qemu_drivedel($vmid, $deviceid); };
4230             warn $@ if $@;
4231             die $err;
4232         }
4233     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4234         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4235         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4236         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4237
4238         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4239
4240         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4241             qemu_iothread_add($vmid, $deviceid, $device);
4242             $devicefull .= ",iothread=iothread-$deviceid";
4243         }
4244
4245         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4246             $devicefull .= ",num_queues=$device->{queues}";
4247         }
4248
4249         qemu_deviceadd($vmid, $devicefull);
4250         qemu_deviceaddverify($vmid, $deviceid);
4251     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4252         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4253         qemu_driveadd($storecfg, $vmid, $device);
4254
4255         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4256         eval { qemu_deviceadd($vmid, $devicefull); };
4257         if (my $err = $@) {
4258             eval { qemu_drivedel($vmid, $deviceid); };
4259             warn $@ if $@;
4260             die $err;
4261         }
4262     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4263         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4264
4265         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4266         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4267         my $use_old_bios_files = undef;
4268         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4269
4270         my $netdevicefull = print_netdevice_full(
4271             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4272         qemu_deviceadd($vmid, $netdevicefull);
4273         eval {
4274             qemu_deviceaddverify($vmid, $deviceid);
4275             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4276         };
4277         if (my $err = $@) {
4278             eval { qemu_netdevdel($vmid, $deviceid); };
4279             warn $@ if $@;
4280             die $err;
4281         }
4282     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4283         my $bridgeid = $2;
4284         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4285         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4286
4287         qemu_deviceadd($vmid, $devicefull);
4288         qemu_deviceaddverify($vmid, $deviceid);
4289     } else {
4290         die "can't hotplug device '$deviceid'\n";
4291     }
4292
4293     return 1;
4294 }
4295
4296 # fixme: this should raise exceptions on error!
4297 sub vm_deviceunplug {
4298     my ($vmid, $conf, $deviceid) = @_;
4299
4300     my $devices_list = vm_devices_list($vmid);
4301     return 1 if !defined($devices_list->{$deviceid});
4302
4303     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4304     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4305
4306     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4307         qemu_devicedel($vmid, $deviceid);
4308     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4309         qemu_devicedel($vmid, $deviceid);
4310         qemu_devicedelverify($vmid, $deviceid);
4311     } elsif ($deviceid =~ m/^usb\d+$/) {
4312         qemu_devicedel($vmid, $deviceid);
4313         qemu_devicedelverify($vmid, $deviceid);
4314     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4315         my $device = parse_drive($deviceid, $conf->{$deviceid});
4316
4317         qemu_devicedel($vmid, $deviceid);
4318         qemu_devicedelverify($vmid, $deviceid);
4319         qemu_drivedel($vmid, $deviceid);
4320         qemu_iothread_del($vmid, $deviceid, $device);
4321     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4322         qemu_devicedel($vmid, $deviceid);
4323         qemu_devicedelverify($vmid, $deviceid);
4324     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4325         my $device = parse_drive($deviceid, $conf->{$deviceid});
4326
4327         qemu_devicedel($vmid, $deviceid);
4328         qemu_devicedelverify($vmid, $deviceid);
4329         qemu_drivedel($vmid, $deviceid);
4330         qemu_deletescsihw($conf, $vmid, $deviceid);
4331
4332         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4333             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4334     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4335         qemu_devicedel($vmid, $deviceid);
4336         qemu_devicedelverify($vmid, $deviceid);
4337         qemu_netdevdel($vmid, $deviceid);
4338     } else {
4339         die "can't unplug device '$deviceid'\n";
4340     }
4341
4342     return 1;
4343 }
4344
4345 sub qemu_spice_usbredir_chardev_add {
4346     my ($vmid, $id) = @_;
4347
4348     mon_cmd($vmid, "chardev-add" , (
4349         id => $id,
4350         backend => {
4351             type => 'spicevmc',
4352             data => {
4353                 type => "usbredir",
4354             },
4355         },
4356     ));
4357 }
4358
4359 sub qemu_iothread_add {
4360     my ($vmid, $deviceid, $device) = @_;
4361
4362     if ($device->{iothread}) {
4363         my $iothreads = vm_iothreads_list($vmid);
4364         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4365     }
4366 }
4367
4368 sub qemu_iothread_del {
4369     my ($vmid, $deviceid, $device) = @_;
4370
4371     if ($device->{iothread}) {
4372         my $iothreads = vm_iothreads_list($vmid);
4373         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4374     }
4375 }
4376
4377 sub qemu_driveadd {
4378     my ($storecfg, $vmid, $device) = @_;
4379
4380     my $kvmver = get_running_qemu_version($vmid);
4381     my $io_uring = min_version($kvmver, 6, 0);
4382     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4383     $drive =~ s/\\/\\\\/g;
4384     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4385
4386     # If the command succeeds qemu prints: "OK"
4387     return 1 if $ret =~ m/OK/s;
4388
4389     die "adding drive failed: $ret\n";
4390 }
4391
4392 sub qemu_drivedel {
4393     my ($vmid, $deviceid) = @_;
4394
4395     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4396     $ret =~ s/^\s+//;
4397
4398     return 1 if $ret eq "";
4399
4400     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4401     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4402
4403     die "deleting drive $deviceid failed : $ret\n";
4404 }
4405
4406 sub qemu_deviceaddverify {
4407     my ($vmid, $deviceid) = @_;
4408
4409     for (my $i = 0; $i <= 5; $i++) {
4410          my $devices_list = vm_devices_list($vmid);
4411          return 1 if defined($devices_list->{$deviceid});
4412          sleep 1;
4413     }
4414
4415     die "error on hotplug device '$deviceid'\n";
4416 }
4417
4418
4419 sub qemu_devicedelverify {
4420     my ($vmid, $deviceid) = @_;
4421
4422     # need to verify that the device is correctly removed as device_del
4423     # is async and empty return is not reliable
4424
4425     for (my $i = 0; $i <= 5; $i++) {
4426          my $devices_list = vm_devices_list($vmid);
4427          return 1 if !defined($devices_list->{$deviceid});
4428          sleep 1;
4429     }
4430
4431     die "error on hot-unplugging device '$deviceid'\n";
4432 }
4433
4434 sub qemu_findorcreatescsihw {
4435     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4436
4437     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4438
4439     my $scsihwid="$controller_prefix$controller";
4440     my $devices_list = vm_devices_list($vmid);
4441
4442     if (!defined($devices_list->{$scsihwid})) {
4443         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4444     }
4445
4446     return 1;
4447 }
4448
4449 sub qemu_deletescsihw {
4450     my ($conf, $vmid, $opt) = @_;
4451
4452     my $device = parse_drive($opt, $conf->{$opt});
4453
4454     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4455         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4456         return 1;
4457     }
4458
4459     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4460
4461     my $devices_list = vm_devices_list($vmid);
4462     foreach my $opt (keys %{$devices_list}) {
4463         if (is_valid_drivename($opt)) {
4464             my $drive = parse_drive($opt, $conf->{$opt});
4465             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4466                 return 1;
4467             }
4468         }
4469     }
4470
4471     my $scsihwid="scsihw$controller";
4472
4473     vm_deviceunplug($vmid, $conf, $scsihwid);
4474
4475     return 1;
4476 }
4477
4478 sub qemu_add_pci_bridge {
4479     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4480
4481     my $bridges = {};
4482
4483     my $bridgeid;
4484
4485     print_pci_addr($device, $bridges, $arch, $machine_type);
4486
4487     while (my ($k, $v) = each %$bridges) {
4488         $bridgeid = $k;
4489     }
4490     return 1 if !defined($bridgeid) || $bridgeid < 1;
4491
4492     my $bridge = "pci.$bridgeid";
4493     my $devices_list = vm_devices_list($vmid);
4494
4495     if (!defined($devices_list->{$bridge})) {
4496         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4497     }
4498
4499     return 1;
4500 }
4501
4502 sub qemu_set_link_status {
4503     my ($vmid, $device, $up) = @_;
4504
4505     mon_cmd($vmid, "set_link", name => $device,
4506                up => $up ? JSON::true : JSON::false);
4507 }
4508
4509 sub qemu_netdevadd {
4510     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4511
4512     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4513     my %options =  split(/[=,]/, $netdev);
4514
4515     if (defined(my $vhost = $options{vhost})) {
4516         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4517     }
4518
4519     if (defined(my $queues = $options{queues})) {
4520         $options{queues} = $queues + 0;
4521     }
4522
4523     mon_cmd($vmid, "netdev_add",  %options);
4524     return 1;
4525 }
4526
4527 sub qemu_netdevdel {
4528     my ($vmid, $deviceid) = @_;
4529
4530     mon_cmd($vmid, "netdev_del", id => $deviceid);
4531 }
4532
4533 sub qemu_usb_hotplug {
4534     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4535
4536     return if !$device;
4537
4538     # remove the old one first
4539     vm_deviceunplug($vmid, $conf, $deviceid);
4540
4541     # check if xhci controller is necessary and available
4542     my $devicelist = vm_devices_list($vmid);
4543
4544     if (!$devicelist->{xhci}) {
4545         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4546         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4547     }
4548
4549     # add the new one
4550     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4551 }
4552
4553 sub qemu_cpu_hotplug {
4554     my ($vmid, $conf, $vcpus) = @_;
4555
4556     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4557
4558     my $sockets = 1;
4559     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4560     $sockets = $conf->{sockets} if  $conf->{sockets};
4561     my $cores = $conf->{cores} || 1;
4562     my $maxcpus = $sockets * $cores;
4563
4564     $vcpus = $maxcpus if !$vcpus;
4565
4566     die "you can't add more vcpus than maxcpus\n"
4567         if $vcpus > $maxcpus;
4568
4569     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4570
4571     if ($vcpus < $currentvcpus) {
4572
4573         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4574
4575             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4576                 qemu_devicedel($vmid, "cpu$i");
4577                 my $retry = 0;
4578                 my $currentrunningvcpus = undef;
4579                 while (1) {
4580                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4581                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4582                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4583                     $retry++;
4584                     sleep 1;
4585                 }
4586                 #update conf after each succesfull cpu unplug
4587                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4588                 PVE::QemuConfig->write_config($vmid, $conf);
4589             }
4590         } else {
4591             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4592         }
4593
4594         return;
4595     }
4596
4597     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4598     die "vcpus in running vm does not match its configuration\n"
4599         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4600
4601     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4602
4603         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4604             my $cpustr = print_cpu_device($conf, $i);
4605             qemu_deviceadd($vmid, $cpustr);
4606
4607             my $retry = 0;
4608             my $currentrunningvcpus = undef;
4609             while (1) {
4610                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4611                 last if scalar(@{$currentrunningvcpus}) == $i;
4612                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4613                 sleep 1;
4614                 $retry++;
4615             }
4616             #update conf after each succesfull cpu hotplug
4617             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4618             PVE::QemuConfig->write_config($vmid, $conf);
4619         }
4620     } else {
4621
4622         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4623             mon_cmd($vmid, "cpu-add", id => int($i));
4624         }
4625     }
4626 }
4627
4628 sub qemu_block_set_io_throttle {
4629     my ($vmid, $deviceid,
4630         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4631         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4632         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4633         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4634
4635     return if !check_running($vmid) ;
4636
4637     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4638         bps => int($bps),
4639         bps_rd => int($bps_rd),
4640         bps_wr => int($bps_wr),
4641         iops => int($iops),
4642         iops_rd => int($iops_rd),
4643         iops_wr => int($iops_wr),
4644         bps_max => int($bps_max),
4645         bps_rd_max => int($bps_rd_max),
4646         bps_wr_max => int($bps_wr_max),
4647         iops_max => int($iops_max),
4648         iops_rd_max => int($iops_rd_max),
4649         iops_wr_max => int($iops_wr_max),
4650         bps_max_length => int($bps_max_length),
4651         bps_rd_max_length => int($bps_rd_max_length),
4652         bps_wr_max_length => int($bps_wr_max_length),
4653         iops_max_length => int($iops_max_length),
4654         iops_rd_max_length => int($iops_rd_max_length),
4655         iops_wr_max_length => int($iops_wr_max_length),
4656     );
4657
4658 }
4659
4660 sub qemu_block_resize {
4661     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4662
4663     my $running = check_running($vmid);
4664
4665     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4666
4667     return if !$running;
4668
4669     my $padding = (1024 - $size % 1024) % 1024;
4670     $size = $size + $padding;
4671
4672     mon_cmd(
4673         $vmid,
4674         "block_resize",
4675         device => $deviceid,
4676         size => int($size),
4677         timeout => 60,
4678     );
4679 }
4680
4681 sub qemu_volume_snapshot {
4682     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4683
4684     my $running = check_running($vmid);
4685
4686     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4687         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4688     } else {
4689         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4690     }
4691 }
4692
4693 sub qemu_volume_snapshot_delete {
4694     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4695
4696     my $running = check_running($vmid);
4697
4698     if($running) {
4699
4700         $running = undef;
4701         my $conf = PVE::QemuConfig->load_config($vmid);
4702         PVE::QemuConfig->foreach_volume($conf, sub {
4703             my ($ds, $drive) = @_;
4704             $running = 1 if $drive->{file} eq $volid;
4705         });
4706     }
4707
4708     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4709         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4710     } else {
4711         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4712     }
4713 }
4714
4715 sub set_migration_caps {
4716     my ($vmid, $savevm) = @_;
4717
4718     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4719
4720     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4721     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4722
4723     my $cap_ref = [];
4724
4725     my $enabled_cap = {
4726         "auto-converge" => 1,
4727         "xbzrle" => 1,
4728         "x-rdma-pin-all" => 0,
4729         "zero-blocks" => 0,
4730         "compress" => 0,
4731         "dirty-bitmaps" => $dirty_bitmaps,
4732     };
4733
4734     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4735
4736     for my $supported_capability (@$supported_capabilities) {
4737         push @$cap_ref, {
4738             capability => $supported_capability->{capability},
4739             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4740         };
4741     }
4742
4743     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4744 }
4745
4746 sub foreach_volid {
4747     my ($conf, $func, @param) = @_;
4748
4749     my $volhash = {};
4750
4751     my $test_volid = sub {
4752         my ($key, $drive, $snapname, $pending) = @_;
4753
4754         my $volid = $drive->{file};
4755         return if !$volid;
4756
4757         $volhash->{$volid}->{cdrom} //= 1;
4758         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4759
4760         my $replicate = $drive->{replicate} // 1;
4761         $volhash->{$volid}->{replicate} //= 0;
4762         $volhash->{$volid}->{replicate} = 1 if $replicate;
4763
4764         $volhash->{$volid}->{shared} //= 0;
4765         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4766
4767         $volhash->{$volid}->{is_unused} //= 0;
4768         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4769
4770         $volhash->{$volid}->{is_attached} //= 0;
4771         $volhash->{$volid}->{is_attached} = 1
4772             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4773
4774         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4775             if defined($snapname);
4776
4777         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4778
4779         my $size = $drive->{size};
4780         $volhash->{$volid}->{size} //= $size if $size;
4781
4782         $volhash->{$volid}->{is_vmstate} //= 0;
4783         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4784
4785         $volhash->{$volid}->{is_tpmstate} //= 0;
4786         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4787
4788         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4789     };
4790
4791     my $include_opts = {
4792         extra_keys => ['vmstate'],
4793         include_unused => 1,
4794     };
4795
4796     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4797
4798     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4799         if defined($conf->{pending}) && $conf->{pending}->%*;
4800
4801     foreach my $snapname (keys %{$conf->{snapshots}}) {
4802         my $snap = $conf->{snapshots}->{$snapname};
4803         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4804     }
4805
4806     foreach my $volid (keys %$volhash) {
4807         &$func($volid, $volhash->{$volid}, @param);
4808     }
4809 }
4810
4811 my $fast_plug_option = {
4812     'description' => 1,
4813     'hookscript' => 1,
4814     'lock' => 1,
4815     'migrate_downtime' => 1,
4816     'migrate_speed' => 1,
4817     'name' => 1,
4818     'onboot' => 1,
4819     'protection' => 1,
4820     'shares' => 1,
4821     'startup' => 1,
4822     'tags' => 1,
4823     'vmstatestorage' => 1,
4824 };
4825
4826 for my $opt (keys %$confdesc_cloudinit) {
4827     $fast_plug_option->{$opt} = 1;
4828 };
4829
4830 # hotplug changes in [PENDING]
4831 # $selection hash can be used to only apply specified options, for
4832 # example: { cores => 1 } (only apply changed 'cores')
4833 # $errors ref is used to return error messages
4834 sub vmconfig_hotplug_pending {
4835     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4836
4837     my $defaults = load_defaults();
4838     my $arch = get_vm_arch($conf);
4839     my $machine_type = get_vm_machine($conf, undef, $arch);
4840
4841     # commit values which do not have any impact on running VM first
4842     # Note: those option cannot raise errors, we we do not care about
4843     # $selection and always apply them.
4844
4845     my $add_error = sub {
4846         my ($opt, $msg) = @_;
4847         $errors->{$opt} = "hotplug problem - $msg";
4848     };
4849
4850     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4851
4852     my $cloudinit_record_changed = sub {
4853         my ($conf, $opt, $old, $new) = @_;
4854         return if !$cloudinit_pending_properties->{$opt};
4855
4856         my $ci = ($conf->{cloudinit} //= {});
4857
4858         my $recorded = $ci->{$opt};
4859         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4860
4861         if (defined($new)) {
4862             if (defined($old)) {
4863                 # an existing value is being modified
4864                 if (defined($recorded)) {
4865                     # the value was already not in sync
4866                     if ($new eq $recorded) {
4867                         # a value is being reverted to the cloud-init state:
4868                         delete $ci->{$opt};
4869                         delete $added{$opt};
4870                     } else {
4871                         # the value was changed multiple times, do nothing
4872                     }
4873                 } elsif ($added{$opt}) {
4874                     # the value had been marked as added and is being changed, do nothing
4875                 } else {
4876                     # the value is new, record it:
4877                     $ci->{$opt} = $old;
4878                 }
4879             } else {
4880                 # a new value is being added
4881                 if (defined($recorded)) {
4882                     # it was already not in sync
4883                     if ($new eq $recorded) {
4884                         # a value is being reverted to the cloud-init state:
4885                         delete $ci->{$opt};
4886                         delete $added{$opt};
4887                     } else {
4888                         # the value had temporarily been removed, do nothing
4889                     }
4890                 } elsif ($added{$opt}) {
4891                     # the value had been marked as added already, do nothing
4892                 } else {
4893                     # the value is new, add it
4894                     $added{$opt} = 1;
4895                 }
4896             }
4897         } elsif (!defined($old)) {
4898             # a non-existent value is being removed? ignore...
4899         } else {
4900             # a value is being deleted
4901             if (defined($recorded)) {
4902                 # a value was already recorded, just keep it
4903             } elsif ($added{$opt}) {
4904                 # the value was marked as added, remove it
4905                 delete $added{$opt};
4906             } else {
4907                 # a previously unrecorded value is being removed, record the old value:
4908                 $ci->{$opt} = $old;
4909             }
4910         }
4911
4912         my $added = join(',', sort keys %added);
4913         $ci->{added} = $added if length($added);
4914     };
4915
4916     my $changes = 0;
4917     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4918         if ($fast_plug_option->{$opt}) {
4919             my $new = delete $conf->{pending}->{$opt};
4920             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4921             $conf->{$opt} = $new;
4922             $changes = 1;
4923         }
4924     }
4925
4926     if ($changes) {
4927         PVE::QemuConfig->write_config($vmid, $conf);
4928     }
4929
4930     my $ostype = $conf->{ostype};
4931     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4932     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4933     my $usb_hotplug = $hotplug_features->{usb}
4934         && min_version($version, 7, 1)
4935         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4936
4937     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4938     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4939
4940     foreach my $opt (sort keys %$pending_delete_hash) {
4941         next if $selection && !$selection->{$opt};
4942         my $force = $pending_delete_hash->{$opt}->{force};
4943         eval {
4944             if ($opt eq 'hotplug') {
4945                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4946             } elsif ($opt eq 'tablet') {
4947                 die "skip\n" if !$hotplug_features->{usb};
4948                 if ($defaults->{tablet}) {
4949                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4950                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4951                         if $arch eq 'aarch64';
4952                 } else {
4953                     vm_deviceunplug($vmid, $conf, 'tablet');
4954                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4955                 }
4956             } elsif ($opt =~ m/^usb(\d+)$/) {
4957                 my $index = $1;
4958                 die "skip\n" if !$usb_hotplug;
4959                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4960                 vm_deviceunplug($vmid, $conf, $opt);
4961             } elsif ($opt eq 'vcpus') {
4962                 die "skip\n" if !$hotplug_features->{cpu};
4963                 qemu_cpu_hotplug($vmid, $conf, undef);
4964             } elsif ($opt eq 'balloon') {
4965                 # enable balloon device is not hotpluggable
4966                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4967                 # here we reset the ballooning value to memory
4968                 my $balloon = get_current_memory($conf->{memory});
4969                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4970             } elsif ($fast_plug_option->{$opt}) {
4971                 # do nothing
4972             } elsif ($opt =~ m/^net(\d+)$/) {
4973                 die "skip\n" if !$hotplug_features->{network};
4974                 vm_deviceunplug($vmid, $conf, $opt);
4975                 if($have_sdn) {
4976                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
4977                     PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
4978                 }
4979             } elsif (is_valid_drivename($opt)) {
4980                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4981                 vm_deviceunplug($vmid, $conf, $opt);
4982                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4983             } elsif ($opt =~ m/^memory$/) {
4984                 die "skip\n" if !$hotplug_features->{memory};
4985                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
4986             } elsif ($opt eq 'cpuunits') {
4987                 $cgroup->change_cpu_shares(undef);
4988             } elsif ($opt eq 'cpulimit') {
4989                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4990             } else {
4991                 die "skip\n";
4992             }
4993         };
4994         if (my $err = $@) {
4995             &$add_error($opt, $err) if $err ne "skip\n";
4996         } else {
4997             my $old = delete $conf->{$opt};
4998             $cloudinit_record_changed->($conf, $opt, $old, undef);
4999             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5000         }
5001     }
5002
5003     my $cloudinit_opt;
5004     foreach my $opt (keys %{$conf->{pending}}) {
5005         next if $selection && !$selection->{$opt};
5006         my $value = $conf->{pending}->{$opt};
5007         eval {
5008             if ($opt eq 'hotplug') {
5009                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5010                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
5011             } elsif ($opt eq 'tablet') {
5012                 die "skip\n" if !$hotplug_features->{usb};
5013                 if ($value == 1) {
5014                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5015                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5016                         if $arch eq 'aarch64';
5017                 } elsif ($value == 0) {
5018                     vm_deviceunplug($vmid, $conf, 'tablet');
5019                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5020                 }
5021             } elsif ($opt =~ m/^usb(\d+)$/) {
5022                 my $index = $1;
5023                 die "skip\n" if !$usb_hotplug;
5024                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5025                 my $id = $opt;
5026                 if ($d->{host} =~ m/^spice$/i)  {
5027                     $id = "usbredirdev$index";
5028                 }
5029                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5030             } elsif ($opt eq 'vcpus') {
5031                 die "skip\n" if !$hotplug_features->{cpu};
5032                 qemu_cpu_hotplug($vmid, $conf, $value);
5033             } elsif ($opt eq 'balloon') {
5034                 # enable/disable balloning device is not hotpluggable
5035                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5036                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5037                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5038
5039                 # allow manual ballooning if shares is set to zero
5040                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5041                     my $memory = get_current_memory($conf->{memory});
5042                     my $balloon = $conf->{pending}->{balloon} || $memory;
5043                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5044                 }
5045             } elsif ($opt =~ m/^net(\d+)$/) {
5046                 # some changes can be done without hotplug
5047                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5048                                     $vmid, $opt, $value, $arch, $machine_type);
5049             } elsif (is_valid_drivename($opt)) {
5050                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5051                 # some changes can be done without hotplug
5052                 my $drive = parse_drive($opt, $value);
5053                 if (drive_is_cloudinit($drive)) {
5054                     $cloudinit_opt = [$opt, $drive];
5055                     # apply all the other changes first, then generate the cloudinit disk
5056                     die "skip\n";
5057                 }
5058                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5059                                      $vmid, $opt, $value, $arch, $machine_type);
5060             } elsif ($opt =~ m/^memory$/) { #dimms
5061                 die "skip\n" if !$hotplug_features->{memory};
5062                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5063             } elsif ($opt eq 'cpuunits') {
5064                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5065                 $cgroup->change_cpu_shares($new_cpuunits);
5066             } elsif ($opt eq 'cpulimit') {
5067                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5068                 $cgroup->change_cpu_quota($cpulimit, 100000);
5069             } elsif ($opt eq 'agent') {
5070                 vmconfig_update_agent($conf, $opt, $value);
5071             } else {
5072                 die "skip\n";  # skip non-hot-pluggable options
5073             }
5074         };
5075         if (my $err = $@) {
5076             &$add_error($opt, $err) if $err ne "skip\n";
5077         } else {
5078             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5079             $conf->{$opt} = $value;
5080             delete $conf->{pending}->{$opt};
5081         }
5082     }
5083
5084     if (defined($cloudinit_opt)) {
5085         my ($opt, $drive) = @$cloudinit_opt;
5086         my $value = $conf->{pending}->{$opt};
5087         eval {
5088             my $temp = {%$conf, $opt => $value};
5089             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5090             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5091                                  $vmid, $opt, $value, $arch, $machine_type);
5092         };
5093         if (my $err = $@) {
5094             &$add_error($opt, $err) if $err ne "skip\n";
5095         } else {
5096             $conf->{$opt} = $value;
5097             delete $conf->{pending}->{$opt};
5098         }
5099     }
5100
5101     # unplug xhci controller if no usb device is left
5102     if ($usb_hotplug) {
5103         my $has_usb = 0;
5104         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5105             next if !defined($conf->{"usb$i"});
5106             $has_usb = 1;
5107             last;
5108         }
5109         if (!$has_usb) {
5110             vm_deviceunplug($vmid, $conf, 'xhci');
5111         }
5112     }
5113
5114     PVE::QemuConfig->write_config($vmid, $conf);
5115
5116     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5117         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5118     }
5119 }
5120
5121 sub try_deallocate_drive {
5122     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5123
5124     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5125         my $volid = $drive->{file};
5126         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5127             my $sid = PVE::Storage::parse_volume_id($volid);
5128             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5129
5130             # check if the disk is really unused
5131             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5132                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5133             PVE::Storage::vdisk_free($storecfg, $volid);
5134             return 1;
5135         } else {
5136             # If vm is not owner of this disk remove from config
5137             return 1;
5138         }
5139     }
5140
5141     return;
5142 }
5143
5144 sub vmconfig_delete_or_detach_drive {
5145     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5146
5147     my $drive = parse_drive($opt, $conf->{$opt});
5148
5149     my $rpcenv = PVE::RPCEnvironment::get();
5150     my $authuser = $rpcenv->get_user();
5151
5152     if ($force) {
5153         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5154         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5155     } else {
5156         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5157     }
5158 }
5159
5160
5161
5162 sub vmconfig_apply_pending {
5163     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5164
5165     return if !scalar(keys %{$conf->{pending}});
5166
5167     my $add_apply_error = sub {
5168         my ($opt, $msg) = @_;
5169         my $err_msg = "unable to apply pending change $opt : $msg";
5170         $errors->{$opt} = $err_msg;
5171         warn $err_msg;
5172     };
5173
5174     # cold plug
5175
5176     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5177     foreach my $opt (sort keys %$pending_delete_hash) {
5178         my $force = $pending_delete_hash->{$opt}->{force};
5179         eval {
5180             if ($opt =~ m/^unused/) {
5181                 die "internal error";
5182             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5183                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5184             } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5185                 if($have_sdn) {
5186                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5187                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5188                     warn if $@;
5189                 }
5190             }
5191         };
5192         if (my $err = $@) {
5193             $add_apply_error->($opt, $err);
5194         } else {
5195             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5196             delete $conf->{$opt};
5197         }
5198     }
5199
5200     PVE::QemuConfig->cleanup_pending($conf);
5201
5202     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5203
5204     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5205         next if $opt eq 'delete'; # just to be sure
5206         eval {
5207             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5208                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5209             } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5210                 if($have_sdn) {
5211                     my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5212                     if ($conf->{$opt}){
5213                         my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5214
5215                         if ($old_net->{bridge} ne $new_net->{bridge} ||
5216                             $old_net->{macaddr} ne $new_net->{macaddr}) {
5217                             PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5218                         }
5219                    }
5220                    #fixme: reuse ip if mac change && same bridge
5221                    PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5222                 }
5223             }
5224         };
5225         if (my $err = $@) {
5226             $add_apply_error->($opt, $err);
5227         } else {
5228
5229             if (is_valid_drivename($opt)) {
5230                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5231                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5232             }
5233
5234             $conf->{$opt} = delete $conf->{pending}->{$opt};
5235         }
5236     }
5237
5238     # write all changes at once to avoid unnecessary i/o
5239     PVE::QemuConfig->write_config($vmid, $conf);
5240     if ($generate_cloudinit) {
5241         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5242             # After successful generation and if there were changes to be applied, update the
5243             # config to drop the {cloudinit} entry.
5244             PVE::QemuConfig->write_config($vmid, $conf);
5245         }
5246     }
5247 }
5248
5249 sub vmconfig_update_net {
5250     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5251
5252     my $newnet = parse_net($value);
5253
5254     if ($conf->{$opt}) {
5255         my $oldnet = parse_net($conf->{$opt});
5256
5257         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5258             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5259             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5260             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5261             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5262
5263             # for non online change, we try to hot-unplug
5264             die "skip\n" if !$hotplug;
5265             vm_deviceunplug($vmid, $conf, $opt);
5266
5267             if($have_sdn) {
5268                 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5269             }
5270
5271         } else {
5272
5273             die "internal error" if $opt !~ m/net(\d+)/;
5274             my $iface = "tap${vmid}i$1";
5275
5276             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5277                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5278                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5279                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5280                 PVE::Network::tap_unplug($iface);
5281
5282                 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5283                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5284                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5285                     qemu_set_link_status($vmid, $opt, 0);
5286                 }
5287
5288                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5289                     if ($have_sdn) {
5290                         PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5291                         PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5292                     }
5293                 }
5294
5295                 if ($have_sdn) {
5296                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5297                 } else {
5298                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5299                 }
5300
5301                 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5302                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5303                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5304                     qemu_set_link_status($vmid, $opt, 1);
5305                 }
5306
5307             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5308                 # Rate can be applied on its own but any change above needs to
5309                 # include the rate in tap_plug since OVS resets everything.
5310                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5311             }
5312
5313             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5314                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5315             }
5316
5317             return 1;
5318         }
5319     }
5320
5321     if ($hotplug) {
5322         if ($have_sdn) {
5323             PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5324             PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5325         }
5326         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5327     } else {
5328         die "skip\n";
5329     }
5330 }
5331
5332 sub vmconfig_update_agent {
5333     my ($conf, $opt, $value) = @_;
5334
5335     die "skip\n" if !$conf->{$opt};
5336
5337     my $hotplug_options = { fstrim_cloned_disks => 1 };
5338
5339     my $old_agent = parse_guest_agent($conf);
5340     my $agent = parse_guest_agent({$opt => $value});
5341
5342     for my $option (keys %$agent) { # added/changed options
5343         next if defined($hotplug_options->{$option});
5344         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5345     }
5346
5347     for my $option (keys %$old_agent) { # removed options
5348         next if defined($hotplug_options->{$option});
5349         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5350     }
5351
5352     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5353 }
5354
5355 sub vmconfig_update_disk {
5356     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5357
5358     my $drive = parse_drive($opt, $value);
5359
5360     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5361         my $media = $drive->{media} || 'disk';
5362         my $oldmedia = $old_drive->{media} || 'disk';
5363         die "unable to change media type\n" if $media ne $oldmedia;
5364
5365         if (!drive_is_cdrom($old_drive)) {
5366
5367             if ($drive->{file} ne $old_drive->{file}) {
5368
5369                 die "skip\n" if !$hotplug;
5370
5371                 # unplug and register as unused
5372                 vm_deviceunplug($vmid, $conf, $opt);
5373                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5374
5375             } else {
5376                 # update existing disk
5377
5378                 # skip non hotpluggable value
5379                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5380                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5381                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5382                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5383                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5384                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5385                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5386                     die "skip\n";
5387                 }
5388
5389                 # apply throttle
5390                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5391                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5392                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5393                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5394                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5395                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5396                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5397                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5398                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5399                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5400                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5401                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5402                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5403                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5404                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5405                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5406                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5407                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5408
5409                     qemu_block_set_io_throttle(
5410                         $vmid,"drive-$opt",
5411                         ($drive->{mbps} || 0)*1024*1024,
5412                         ($drive->{mbps_rd} || 0)*1024*1024,
5413                         ($drive->{mbps_wr} || 0)*1024*1024,
5414                          $drive->{iops} || 0,
5415                          $drive->{iops_rd} || 0,
5416                          $drive->{iops_wr} || 0,
5417                         ($drive->{mbps_max} || 0)*1024*1024,
5418                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5419                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5420                          $drive->{iops_max} || 0,
5421                          $drive->{iops_rd_max} || 0,
5422                          $drive->{iops_wr_max} || 0,
5423                          $drive->{bps_max_length} || 1,
5424                          $drive->{bps_rd_max_length} || 1,
5425                          $drive->{bps_wr_max_length} || 1,
5426                          $drive->{iops_max_length} || 1,
5427                          $drive->{iops_rd_max_length} || 1,
5428                          $drive->{iops_wr_max_length} || 1,
5429                     );
5430
5431                 }
5432
5433                 return 1;
5434             }
5435
5436         } else { # cdrom
5437
5438             if ($drive->{file} eq 'none') {
5439                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5440                 if (drive_is_cloudinit($old_drive)) {
5441                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5442                 }
5443             } else {
5444                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5445
5446                 # force eject if locked
5447                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5448
5449                 if ($path) {
5450                     mon_cmd($vmid, "blockdev-change-medium",
5451                         id => "$opt", filename => "$path");
5452                 }
5453             }
5454
5455             return 1;
5456         }
5457     }
5458
5459     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5460     # hotplug new disks
5461     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5462     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5463 }
5464
5465 sub vmconfig_update_cloudinit_drive {
5466     my ($storecfg, $conf, $vmid) = @_;
5467
5468     my $cloudinit_ds = undef;
5469     my $cloudinit_drive = undef;
5470
5471     PVE::QemuConfig->foreach_volume($conf, sub {
5472         my ($ds, $drive) = @_;
5473         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5474             $cloudinit_ds = $ds;
5475             $cloudinit_drive = $drive;
5476         }
5477     });
5478
5479     return if !$cloudinit_drive;
5480
5481     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5482         PVE::QemuConfig->write_config($vmid, $conf);
5483     }
5484
5485     my $running = PVE::QemuServer::check_running($vmid);
5486
5487     if ($running) {
5488         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5489         if ($path) {
5490             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5491             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5492         }
5493     }
5494 }
5495
5496 # called in locked context by incoming migration
5497 sub vm_migrate_get_nbd_disks {
5498     my ($storecfg, $conf, $replicated_volumes) = @_;
5499
5500     my $local_volumes = {};
5501     PVE::QemuConfig->foreach_volume($conf, sub {
5502         my ($ds, $drive) = @_;
5503
5504         return if drive_is_cdrom($drive);
5505         return if $ds eq 'tpmstate0';
5506
5507         my $volid = $drive->{file};
5508
5509         return if !$volid;
5510
5511         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5512
5513         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5514         return if $scfg->{shared};
5515
5516         my $format = qemu_img_format($scfg, $volname);
5517
5518         # replicated disks re-use existing state via bitmap
5519         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5520         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5521     });
5522     return $local_volumes;
5523 }
5524
5525 # called in locked context by incoming migration
5526 sub vm_migrate_alloc_nbd_disks {
5527     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5528
5529     my $nbd = {};
5530     foreach my $opt (sort keys %$source_volumes) {
5531         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5532
5533         if ($use_existing) {
5534             $nbd->{$opt}->{drivestr} = print_drive($drive);
5535             $nbd->{$opt}->{volid} = $volid;
5536             $nbd->{$opt}->{replicated} = 1;
5537             next;
5538         }
5539
5540         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5541
5542         # order of precedence, filtered by whether storage supports it:
5543         # 1. explicit requested format
5544         # 2. default format of storage
5545         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5546         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5547
5548         my $size = $drive->{size} / 1024;
5549         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5550         my $newdrive = $drive;
5551         $newdrive->{format} = $format;
5552         $newdrive->{file} = $newvolid;
5553         my $drivestr = print_drive($newdrive);
5554         $nbd->{$opt}->{drivestr} = $drivestr;
5555         $nbd->{$opt}->{volid} = $newvolid;
5556     }
5557
5558     return $nbd;
5559 }
5560
5561 # see vm_start_nolock for parameters, additionally:
5562 # migrate_opts:
5563 #   storagemap = parsed storage map for allocating NBD disks
5564 sub vm_start {
5565     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5566
5567     return PVE::QemuConfig->lock_config($vmid, sub {
5568         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5569
5570         die "you can't start a vm if it's a template\n"
5571             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5572
5573         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5574         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5575
5576         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5577
5578         if ($has_backup_lock && $running) {
5579             # a backup is currently running, attempt to start the guest in the
5580             # existing QEMU instance
5581             return vm_resume($vmid);
5582         }
5583
5584         PVE::QemuConfig->check_lock($conf)
5585             if !($params->{skiplock} || $has_suspended_lock);
5586
5587         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5588
5589         die "VM $vmid already running\n" if $running;
5590
5591         if (my $storagemap = $migrate_opts->{storagemap}) {
5592             my $replicated = $migrate_opts->{replicated_volumes};
5593             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5594             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5595
5596             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5597                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5598             }
5599         }
5600
5601         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5602     });
5603 }
5604
5605
5606 # params:
5607 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5608 #   skiplock => 0/1, skip checking for config lock
5609 #   skiptemplate => 0/1, skip checking whether VM is template
5610 #   forcemachine => to force QEMU machine (rollback/migration)
5611 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5612 #   timeout => in seconds
5613 #   paused => start VM in paused state (backup)
5614 #   resume => resume from hibernation
5615 #   pbs-backing => {
5616 #      sata0 => {
5617 #         repository
5618 #         snapshot
5619 #         keyfile
5620 #         archive
5621 #      },
5622 #      virtio2 => ...
5623 #   }
5624 # migrate_opts:
5625 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5626 #   migratedfrom => source node
5627 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5628 #   network => CIDR of migration network
5629 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5630 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5631 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5632 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5633 #       contained in config
5634 sub vm_start_nolock {
5635     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5636
5637     my $statefile = $params->{statefile};
5638     my $resume = $params->{resume};
5639
5640     my $migratedfrom = $migrate_opts->{migratedfrom};
5641     my $migration_type = $migrate_opts->{type};
5642
5643     my $res = {};
5644
5645     # clean up leftover reboot request files
5646     eval { clear_reboot_request($vmid); };
5647     warn $@ if $@;
5648
5649     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5650         vmconfig_apply_pending($vmid, $conf, $storecfg);
5651         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5652     }
5653
5654     # don't regenerate the ISO if the VM is started as part of a live migration
5655     # this way we can reuse the old ISO with the correct config
5656     if (!$migratedfrom) {
5657         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5658             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5659             # $conf->{cloudinit}, so we could just not do this?
5660             # But we do it above, so for now let's be consistent.
5661             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5662         }
5663     }
5664
5665     # override offline migrated volumes, conf is out of date still
5666     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5667         for my $key (sort keys $offline_volumes->%*) {
5668             my $parsed = parse_drive($key, $conf->{$key});
5669             $parsed->{file} = $offline_volumes->{$key};
5670             $conf->{$key} = print_drive($parsed);
5671         }
5672     }
5673
5674     my $defaults = load_defaults();
5675
5676     # set environment variable useful inside network script
5677     # for remote migration the config is available on the target node!
5678     if (!$migrate_opts->{remote_node}) {
5679         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5680     }
5681
5682     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5683
5684     my $forcemachine = $params->{forcemachine};
5685     my $forcecpu = $params->{forcecpu};
5686     if ($resume) {
5687         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5688         $forcemachine = $conf->{runningmachine};
5689         $forcecpu = $conf->{runningcpu};
5690         print "Resuming suspended VM\n";
5691     }
5692
5693     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5694         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5695
5696     my $migration_ip;
5697     my $get_migration_ip = sub {
5698         my ($nodename) = @_;
5699
5700         return $migration_ip if defined($migration_ip);
5701
5702         my $cidr = $migrate_opts->{network};
5703
5704         if (!defined($cidr)) {
5705             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5706             $cidr = $dc_conf->{migration}->{network};
5707         }
5708
5709         if (defined($cidr)) {
5710             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5711
5712             die "could not get IP: no address configured on local " .
5713                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5714
5715             die "could not get IP: multiple addresses configured on local " .
5716                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5717
5718             $migration_ip = @$ips[0];
5719         }
5720
5721         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5722             if !defined($migration_ip);
5723
5724         return $migration_ip;
5725     };
5726
5727     if ($statefile) {
5728         if ($statefile eq 'tcp') {
5729             my $migrate = $res->{migrate} = { proto => 'tcp' };
5730             $migrate->{addr} = "localhost";
5731             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5732             my $nodename = nodename();
5733
5734             if (!defined($migration_type)) {
5735                 if (defined($datacenterconf->{migration}->{type})) {
5736                     $migration_type = $datacenterconf->{migration}->{type};
5737                 } else {
5738                     $migration_type = 'secure';
5739                 }
5740             }
5741
5742             if ($migration_type eq 'insecure') {
5743                 $migrate->{addr} = $get_migration_ip->($nodename);
5744                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5745             }
5746
5747             # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5748             # via QMP later
5749             push @$cmd, '-incoming', 'defer';
5750             push @$cmd, '-S';
5751
5752         } elsif ($statefile eq 'unix') {
5753             # should be default for secure migrations as a ssh TCP forward
5754             # tunnel is not deterministic reliable ready and fails regurarly
5755             # to set up in time, so use UNIX socket forwards
5756             my $migrate = $res->{migrate} = { proto => 'unix' };
5757             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5758             unlink $migrate->{addr};
5759
5760             $migrate->{uri} = "unix:$migrate->{addr}";
5761             push @$cmd, '-incoming', $migrate->{uri};
5762             push @$cmd, '-S';
5763
5764         } elsif (-e $statefile) {
5765             push @$cmd, '-loadstate', $statefile;
5766         } else {
5767             my $statepath = PVE::Storage::path($storecfg, $statefile);
5768             push @$vollist, $statefile;
5769             push @$cmd, '-loadstate', $statepath;
5770         }
5771     } elsif ($params->{paused}) {
5772         push @$cmd, '-S';
5773     }
5774
5775     my $memory = get_current_memory($conf->{memory});
5776     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5777
5778     my $pci_reserve_list = [];
5779     for my $device (values $pci_devices->%*) {
5780         next if $device->{mdev}; # we don't reserve for mdev devices
5781         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5782     }
5783
5784     # reserve all PCI IDs before actually doing anything with them
5785     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5786
5787     eval {
5788         my $uuid;
5789         for my $id (sort keys %$pci_devices) {
5790             my $d = $pci_devices->{$id};
5791             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5792
5793             my $chosen_mdev;
5794             for my $dev ($d->{ids}->@*) {
5795                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5796                 if ($d->{mdev}) {
5797                     warn $@ if $@;
5798                     $chosen_mdev = $info;
5799                     last if $chosen_mdev; # if successful, we're done
5800                 } else {
5801                     die $@ if $@;
5802                 }
5803             }
5804
5805             next if !$d->{mdev};
5806             die "could not create mediated device\n" if !defined($chosen_mdev);
5807
5808             # nvidia grid needs the uuid of the mdev as qemu parameter
5809             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5810                 if (defined($conf->{smbios1})) {
5811                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5812                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5813                 }
5814                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5815             }
5816         }
5817         push @$cmd, '-uuid', $uuid if defined($uuid);
5818     };
5819     if (my $err = $@) {
5820         eval { cleanup_pci_devices($vmid, $conf) };
5821         warn $@ if $@;
5822         die $err;
5823     }
5824
5825     PVE::Storage::activate_volumes($storecfg, $vollist);
5826
5827
5828     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5829     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5830     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5831     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5832     # timeout should be more than enough here...
5833     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5834
5835     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5836
5837     my %run_params = (
5838         timeout => $statefile ? undef : $start_timeout,
5839         umask => 0077,
5840         noerr => 1,
5841     );
5842
5843     # when migrating, prefix QEMU output so other side can pick up any
5844     # errors that might occur and show the user
5845     if ($migratedfrom) {
5846         $run_params{quiet} = 1;
5847         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5848     }
5849
5850     my %systemd_properties = (
5851         Slice => 'qemu.slice',
5852         KillMode => 'process',
5853         SendSIGKILL => 0,
5854         TimeoutStopUSec => ULONG_MAX, # infinity
5855     );
5856
5857     if (PVE::CGroup::cgroup_mode() == 2) {
5858         $systemd_properties{CPUWeight} = $cpuunits;
5859     } else {
5860         $systemd_properties{CPUShares} = $cpuunits;
5861     }
5862
5863     if (my $cpulimit = $conf->{cpulimit}) {
5864         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5865     }
5866     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5867
5868     my $run_qemu = sub {
5869         PVE::Tools::run_fork sub {
5870             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5871
5872             my $tpmpid;
5873             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5874                 # start the TPM emulator so QEMU can connect on start
5875                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5876             }
5877
5878             my $exitcode = run_command($cmd, %run_params);
5879             if ($exitcode) {
5880                 if ($tpmpid) {
5881                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5882                     kill 'TERM', $tpmpid;
5883                 }
5884                 die "QEMU exited with code $exitcode\n";
5885             }
5886         };
5887     };
5888
5889     if ($conf->{hugepages}) {
5890
5891         my $code = sub {
5892             my $hotplug_features =
5893                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5894             my $hugepages_topology =
5895                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5896
5897             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5898
5899             PVE::QemuServer::Memory::hugepages_mount();
5900             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5901
5902             eval { $run_qemu->() };
5903             if (my $err = $@) {
5904                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5905                     if !$conf->{keephugepages};
5906                 die $err;
5907             }
5908
5909             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5910                 if !$conf->{keephugepages};
5911         };
5912         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5913
5914     } else {
5915         eval { $run_qemu->() };
5916     }
5917
5918     if (my $err = $@) {
5919         # deactivate volumes if start fails
5920         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5921         warn $@ if $@;
5922         eval { cleanup_pci_devices($vmid, $conf) };
5923         warn $@ if $@;
5924
5925         die "start failed: $err";
5926     }
5927
5928     # re-reserve all PCI IDs now that we can know the actual VM PID
5929     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5930     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5931     warn $@ if $@;
5932
5933     if (defined(my $migrate = $res->{migrate})) {
5934         if ($migrate->{proto} eq 'tcp') {
5935             my $nodename = nodename();
5936             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5937             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5938             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5939             mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
5940         }
5941         print "migration listens on $migrate->{uri}\n";
5942     } elsif ($statefile) {
5943         eval { mon_cmd($vmid, "cont"); };
5944         warn $@ if $@;
5945     }
5946
5947     #start nbd server for storage migration
5948     if (my $nbd = $migrate_opts->{nbd}) {
5949         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5950
5951         my $migrate_storage_uri;
5952         # nbd_protocol_version > 0 for unix socket support
5953         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5954             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5955             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5956             $migrate_storage_uri = "nbd:unix:$socket_path";
5957             $res->{migrate}->{unix_sockets} = [$socket_path];
5958         } else {
5959             my $nodename = nodename();
5960             my $localip = $get_migration_ip->($nodename);
5961             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5962             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5963
5964             mon_cmd($vmid, "nbd-server-start", addr => {
5965                 type => 'inet',
5966                 data => {
5967                     host => "${localip}",
5968                     port => "${storage_migrate_port}",
5969                 },
5970             });
5971             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5972             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5973         }
5974
5975         my $block_info = mon_cmd($vmid, "query-block");
5976         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5977
5978         foreach my $opt (sort keys %$nbd) {
5979             my $drivestr = $nbd->{$opt}->{drivestr};
5980             my $volid = $nbd->{$opt}->{volid};
5981
5982             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5983
5984             mon_cmd(
5985                 $vmid,
5986                 "block-export-add",
5987                 id => "drive-$opt",
5988                 'node-name' => $block_node,
5989                 writable => JSON::true,
5990                 type => "nbd",
5991                 name => "drive-$opt", # NBD export name
5992             );
5993
5994             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5995             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5996             print "re-using replicated volume: $opt - $volid\n"
5997                 if $nbd->{$opt}->{replicated};
5998
5999             $res->{drives}->{$opt} = $nbd->{$opt};
6000             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6001         }
6002     }
6003
6004     if ($migratedfrom) {
6005         eval {
6006             set_migration_caps($vmid);
6007         };
6008         warn $@ if $@;
6009
6010         if ($spice_port) {
6011             print "spice listens on port $spice_port\n";
6012             $res->{spice_port} = $spice_port;
6013             if ($migrate_opts->{spice_ticket}) {
6014                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6015                     $migrate_opts->{spice_ticket});
6016                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6017             }
6018         }
6019
6020     } else {
6021         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6022             if !$statefile && $conf->{balloon};
6023
6024         foreach my $opt (keys %$conf) {
6025             next if $opt !~  m/^net\d+$/;
6026             my $nicconf = parse_net($conf->{$opt});
6027             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6028         }
6029         add_nets_bridge_fdb($conf, $vmid);
6030     }
6031
6032    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6033         eval {
6034             mon_cmd(
6035                 $vmid,
6036                 'qom-set',
6037                 path => "machine/peripheral/balloon0",
6038                 property => "guest-stats-polling-interval",
6039                 value => 2
6040             );
6041         };
6042         log_warn("could not set polling interval for ballooning - $@") if $@;
6043     }
6044
6045     if ($resume) {
6046         print "Resumed VM, removing state\n";
6047         if (my $vmstate = $conf->{vmstate}) {
6048             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6049             PVE::Storage::vdisk_free($storecfg, $vmstate);
6050         }
6051         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6052         PVE::QemuConfig->write_config($vmid, $conf);
6053     }
6054
6055     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6056
6057     my ($current_machine, $is_deprecated) =
6058         PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6059     if ($is_deprecated) {
6060         log_warn(
6061             "current machine version '$current_machine' is deprecated - see the documentation and ".
6062             "change to a newer one",
6063         );
6064     }
6065
6066     return $res;
6067 }
6068
6069 sub vm_commandline {
6070     my ($storecfg, $vmid, $snapname) = @_;
6071
6072     my $conf = PVE::QemuConfig->load_config($vmid);
6073
6074     my ($forcemachine, $forcecpu);
6075     if ($snapname) {
6076         my $snapshot = $conf->{snapshots}->{$snapname};
6077         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6078
6079         # check for machine or CPU overrides in snapshot
6080         $forcemachine = $snapshot->{runningmachine};
6081         $forcecpu = $snapshot->{runningcpu};
6082
6083         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6084
6085         $conf = $snapshot;
6086     }
6087
6088     my $defaults = load_defaults();
6089
6090     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6091
6092     return PVE::Tools::cmd2string($cmd);
6093 }
6094
6095 sub vm_reset {
6096     my ($vmid, $skiplock) = @_;
6097
6098     PVE::QemuConfig->lock_config($vmid, sub {
6099
6100         my $conf = PVE::QemuConfig->load_config($vmid);
6101
6102         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6103
6104         mon_cmd($vmid, "system_reset");
6105     });
6106 }
6107
6108 sub get_vm_volumes {
6109     my ($conf) = @_;
6110
6111     my $vollist = [];
6112     foreach_volid($conf, sub {
6113         my ($volid, $attr) = @_;
6114
6115         return if $volid =~ m|^/|;
6116
6117         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6118         return if !$sid;
6119
6120         push @$vollist, $volid;
6121     });
6122
6123     return $vollist;
6124 }
6125
6126 sub cleanup_pci_devices {
6127     my ($vmid, $conf) = @_;
6128
6129     foreach my $key (keys %$conf) {
6130         next if $key !~ m/^hostpci(\d+)$/;
6131         my $hostpciindex = $1;
6132         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6133         my $d = parse_hostpci($conf->{$key});
6134         if ($d->{mdev}) {
6135             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6136             # don't want to break ABI just for this two liner
6137             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6138
6139             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6140             # out when we do it first. so wait for 10 seconds and then try it
6141             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/) {
6142                 sleep 10;
6143             }
6144
6145             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6146         }
6147     }
6148     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6149 }
6150
6151 sub vm_stop_cleanup {
6152     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6153
6154     eval {
6155
6156         if (!$keepActive) {
6157             my $vollist = get_vm_volumes($conf);
6158             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6159
6160             if (my $tpmdrive = $conf->{tpmstate0}) {
6161                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6162                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6163                 if ($storeid) {
6164                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6165                 }
6166             }
6167         }
6168
6169         foreach my $ext (qw(mon qmp pid vnc qga)) {
6170             unlink "/var/run/qemu-server/${vmid}.$ext";
6171         }
6172
6173         if ($conf->{ivshmem}) {
6174             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6175             # just delete it for now, VMs which have this already open do not
6176             # are affected, but new VMs will get a separated one. If this
6177             # becomes an issue we either add some sort of ref-counting or just
6178             # add a "don't delete on stop" flag to the ivshmem format.
6179             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6180         }
6181
6182         cleanup_pci_devices($vmid, $conf);
6183
6184         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6185     };
6186     warn $@ if $@; # avoid errors - just warn
6187 }
6188
6189 # call only in locked context
6190 sub _do_vm_stop {
6191     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6192
6193     my $pid = check_running($vmid, $nocheck);
6194     return if !$pid;
6195
6196     my $conf;
6197     if (!$nocheck) {
6198         $conf = PVE::QemuConfig->load_config($vmid);
6199         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6200         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6201             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6202             $timeout = $opts->{down} if $opts->{down};
6203         }
6204         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6205     }
6206
6207     eval {
6208         if ($shutdown) {
6209             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6210                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6211             } else {
6212                 mon_cmd($vmid, "system_powerdown");
6213             }
6214         } else {
6215             mon_cmd($vmid, "quit");
6216         }
6217     };
6218     my $err = $@;
6219
6220     if (!$err) {
6221         $timeout = 60 if !defined($timeout);
6222
6223         my $count = 0;
6224         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6225             $count++;
6226             sleep 1;
6227         }
6228
6229         if ($count >= $timeout) {
6230             if ($force) {
6231                 warn "VM still running - terminating now with SIGTERM\n";
6232                 kill 15, $pid;
6233             } else {
6234                 die "VM quit/powerdown failed - got timeout\n";
6235             }
6236         } else {
6237             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6238             return;
6239         }
6240     } else {
6241        if (!check_running($vmid, $nocheck)) {
6242             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6243             return;
6244        }
6245        if ($force) {
6246             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6247             kill 15, $pid;
6248         } else {
6249             die "VM quit/powerdown failed\n";
6250         }
6251     }
6252
6253     # wait again
6254     $timeout = 10;
6255
6256     my $count = 0;
6257     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6258         $count++;
6259         sleep 1;
6260     }
6261
6262     if ($count >= $timeout) {
6263         warn "VM still running - terminating now with SIGKILL\n";
6264         kill 9, $pid;
6265         sleep 1;
6266     }
6267
6268     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6269 }
6270
6271 # Note: use $nocheck to skip tests if VM configuration file exists.
6272 # We need that when migration VMs to other nodes (files already moved)
6273 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6274 sub vm_stop {
6275     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6276
6277     $force = 1 if !defined($force) && !$shutdown;
6278
6279     if ($migratedfrom){
6280         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6281         kill 15, $pid if $pid;
6282         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6283         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6284         return;
6285     }
6286
6287     PVE::QemuConfig->lock_config($vmid, sub {
6288         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6289    });
6290 }
6291
6292 sub vm_reboot {
6293     my ($vmid, $timeout) = @_;
6294
6295     PVE::QemuConfig->lock_config($vmid, sub {
6296         eval {
6297
6298             # only reboot if running, as qmeventd starts it again on a stop event
6299             return if !check_running($vmid);
6300
6301             create_reboot_request($vmid);
6302
6303             my $storecfg = PVE::Storage::config();
6304             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6305
6306         };
6307         if (my $err = $@) {
6308             # avoid that the next normal shutdown will be confused for a reboot
6309             clear_reboot_request($vmid);
6310             die $err;
6311         }
6312    });
6313 }
6314
6315 # note: if using the statestorage parameter, the caller has to check privileges
6316 sub vm_suspend {
6317     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6318
6319     my $conf;
6320     my $path;
6321     my $storecfg;
6322     my $vmstate;
6323
6324     PVE::QemuConfig->lock_config($vmid, sub {
6325
6326         $conf = PVE::QemuConfig->load_config($vmid);
6327
6328         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6329         PVE::QemuConfig->check_lock($conf)
6330             if !($skiplock || $is_backing_up);
6331
6332         die "cannot suspend to disk during backup\n"
6333             if $is_backing_up && $includestate;
6334
6335         if ($includestate) {
6336             $conf->{lock} = 'suspending';
6337             my $date = strftime("%Y-%m-%d", localtime(time()));
6338             $storecfg = PVE::Storage::config();
6339             if (!$statestorage) {
6340                 $statestorage = find_vmstate_storage($conf, $storecfg);
6341                 # check permissions for the storage
6342                 my $rpcenv = PVE::RPCEnvironment::get();
6343                 if ($rpcenv->{type} ne 'cli') {
6344                     my $authuser = $rpcenv->get_user();
6345                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6346                 }
6347             }
6348
6349
6350             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6351                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6352             $path = PVE::Storage::path($storecfg, $vmstate);
6353             PVE::QemuConfig->write_config($vmid, $conf);
6354         } else {
6355             mon_cmd($vmid, "stop");
6356         }
6357     });
6358
6359     if ($includestate) {
6360         # save vm state
6361         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6362
6363         eval {
6364             set_migration_caps($vmid, 1);
6365             mon_cmd($vmid, "savevm-start", statefile => $path);
6366             for(;;) {
6367                 my $state = mon_cmd($vmid, "query-savevm");
6368                 if (!$state->{status}) {
6369                     die "savevm not active\n";
6370                 } elsif ($state->{status} eq 'active') {
6371                     sleep(1);
6372                     next;
6373                 } elsif ($state->{status} eq 'completed') {
6374                     print "State saved, quitting\n";
6375                     last;
6376                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6377                     die "query-savevm failed with error '$state->{error}'\n"
6378                 } else {
6379                     die "query-savevm returned status '$state->{status}'\n";
6380                 }
6381             }
6382         };
6383         my $err = $@;
6384
6385         PVE::QemuConfig->lock_config($vmid, sub {
6386             $conf = PVE::QemuConfig->load_config($vmid);
6387             if ($err) {
6388                 # cleanup, but leave suspending lock, to indicate something went wrong
6389                 eval {
6390                     mon_cmd($vmid, "savevm-end");
6391                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6392                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6393                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6394                     PVE::QemuConfig->write_config($vmid, $conf);
6395                 };
6396                 warn $@ if $@;
6397                 die $err;
6398             }
6399
6400             die "lock changed unexpectedly\n"
6401                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6402
6403             mon_cmd($vmid, "quit");
6404             $conf->{lock} = 'suspended';
6405             PVE::QemuConfig->write_config($vmid, $conf);
6406         });
6407     }
6408 }
6409
6410 # $nocheck is set when called as part of a migration - in this context the
6411 # location of the config file (source or target node) is not deterministic,
6412 # since migration cannot wait for pmxcfs to process the rename
6413 sub vm_resume {
6414     my ($vmid, $skiplock, $nocheck) = @_;
6415
6416     PVE::QemuConfig->lock_config($vmid, sub {
6417         my $res = mon_cmd($vmid, 'query-status');
6418         my $resume_cmd = 'cont';
6419         my $reset = 0;
6420         my $conf;
6421         if ($nocheck) {
6422             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6423             if ($@) {
6424                 my $vmlist = PVE::Cluster::get_vmlist();
6425                 if (exists($vmlist->{ids}->{$vmid})) {
6426                     my $node = $vmlist->{ids}->{$vmid}->{node};
6427                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6428                 }
6429                 if (!$conf) {
6430                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6431                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6432                 }
6433             }
6434         } else {
6435             $conf = PVE::QemuConfig->load_config($vmid);
6436         }
6437
6438         if ($res->{status}) {
6439             return if $res->{status} eq 'running'; # job done, go home
6440             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6441             $reset = 1 if $res->{status} eq 'shutdown';
6442         }
6443
6444         if (!$nocheck) {
6445             PVE::QemuConfig->check_lock($conf)
6446                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6447         }
6448
6449         if ($reset) {
6450             # required if a VM shuts down during a backup and we get a resume
6451             # request before the backup finishes for example
6452             mon_cmd($vmid, "system_reset");
6453         }
6454
6455         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6456
6457         mon_cmd($vmid, $resume_cmd);
6458     });
6459 }
6460
6461 sub vm_sendkey {
6462     my ($vmid, $skiplock, $key) = @_;
6463
6464     PVE::QemuConfig->lock_config($vmid, sub {
6465
6466         my $conf = PVE::QemuConfig->load_config($vmid);
6467
6468         # there is no qmp command, so we use the human monitor command
6469         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6470         die $res if $res ne '';
6471     });
6472 }
6473
6474 sub check_bridge_access {
6475     my ($rpcenv, $authuser, $conf) = @_;
6476
6477     return 1 if $authuser eq 'root@pam';
6478
6479     for my $opt (sort keys $conf->%*) {
6480         next if $opt !~ m/^net\d+$/;
6481         my $net = parse_net($conf->{$opt});
6482         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6483         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6484     }
6485     return 1;
6486 };
6487
6488 sub check_mapping_access {
6489     my ($rpcenv, $user, $conf) = @_;
6490
6491     for my $opt (keys $conf->%*) {
6492         if ($opt =~ m/^usb\d+$/) {
6493             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6494             if (my $host = $device->{host}) {
6495                 die "only root can set '$opt' config for real devices\n"
6496                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6497             } elsif ($device->{mapping}) {
6498                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6499             } else {
6500                 die "either 'host' or 'mapping' must be set.\n";
6501             }
6502         } elsif ($opt =~ m/^hostpci\d+$/) {
6503             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6504             if ($device->{host}) {
6505                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6506             } elsif ($device->{mapping}) {
6507                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6508             } else {
6509                 die "either 'host' or 'mapping' must be set.\n";
6510             }
6511        }
6512    }
6513 };
6514
6515 sub check_restore_permissions {
6516     my ($rpcenv, $user, $conf) = @_;
6517
6518     check_bridge_access($rpcenv, $user, $conf);
6519     check_mapping_access($rpcenv, $user, $conf);
6520 }
6521 # vzdump restore implementaion
6522
6523 sub tar_archive_read_firstfile {
6524     my $archive = shift;
6525
6526     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6527
6528     # try to detect archive type first
6529     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6530         die "unable to open file '$archive'\n";
6531     my $firstfile = <$fh>;
6532     kill 15, $pid;
6533     close $fh;
6534
6535     die "ERROR: archive contaions no data\n" if !$firstfile;
6536     chomp $firstfile;
6537
6538     return $firstfile;
6539 }
6540
6541 sub tar_restore_cleanup {
6542     my ($storecfg, $statfile) = @_;
6543
6544     print STDERR "starting cleanup\n";
6545
6546     if (my $fd = IO::File->new($statfile, "r")) {
6547         while (defined(my $line = <$fd>)) {
6548             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6549                 my $volid = $2;
6550                 eval {
6551                     if ($volid =~ m|^/|) {
6552                         unlink $volid || die 'unlink failed\n';
6553                     } else {
6554                         PVE::Storage::vdisk_free($storecfg, $volid);
6555                     }
6556                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6557                 };
6558                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6559             } else {
6560                 print STDERR "unable to parse line in statfile - $line";
6561             }
6562         }
6563         $fd->close();
6564     }
6565 }
6566
6567 sub restore_file_archive {
6568     my ($archive, $vmid, $user, $opts) = @_;
6569
6570     return restore_vma_archive($archive, $vmid, $user, $opts)
6571         if $archive eq '-';
6572
6573     my $info = PVE::Storage::archive_info($archive);
6574     my $format = $opts->{format} // $info->{format};
6575     my $comp = $info->{compression};
6576
6577     # try to detect archive format
6578     if ($format eq 'tar') {
6579         return restore_tar_archive($archive, $vmid, $user, $opts);
6580     } else {
6581         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6582     }
6583 }
6584
6585 # hepler to remove disks that will not be used after restore
6586 my $restore_cleanup_oldconf = sub {
6587     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6588
6589     my $kept_disks = {};
6590
6591     PVE::QemuConfig->foreach_volume($oldconf, sub {
6592         my ($ds, $drive) = @_;
6593
6594         return if drive_is_cdrom($drive, 1);
6595
6596         my $volid = $drive->{file};
6597         return if !$volid || $volid =~ m|^/|;
6598
6599         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6600         return if !$path || !$owner || ($owner != $vmid);
6601
6602         # Note: only delete disk we want to restore
6603         # other volumes will become unused
6604         if ($virtdev_hash->{$ds}) {
6605             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6606             if (my $err = $@) {
6607                 warn $err;
6608             }
6609         } else {
6610             $kept_disks->{$volid} = 1;
6611         }
6612     });
6613
6614     # after the restore we have no snapshots anymore
6615     for my $snapname (keys $oldconf->{snapshots}->%*) {
6616         my $snap = $oldconf->{snapshots}->{$snapname};
6617         if ($snap->{vmstate}) {
6618             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6619             if (my $err = $@) {
6620                 warn $err;
6621             }
6622         }
6623
6624         for my $volid (keys $kept_disks->%*) {
6625             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6626             warn $@ if $@;
6627         }
6628     }
6629 };
6630
6631 # Helper to parse vzdump backup device hints
6632 #
6633 # $rpcenv: Environment, used to ckeck storage permissions
6634 # $user: User ID, to check storage permissions
6635 # $storecfg: Storage configuration
6636 # $fh: the file handle for reading the configuration
6637 # $devinfo: should contain device sizes for all backu-up'ed devices
6638 # $options: backup options (pool, default storage)
6639 #
6640 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6641 my $parse_backup_hints = sub {
6642     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6643
6644     my $check_storage = sub { # assert if an image can be allocate
6645         my ($storeid, $scfg) = @_;
6646         die "Content type 'images' is not available on storage '$storeid'\n"
6647             if !$scfg->{content}->{images};
6648         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6649             if $user ne 'root@pam';
6650     };
6651
6652     my $virtdev_hash = {};
6653     while (defined(my $line = <$fh>)) {
6654         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6655             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6656             die "archive does not contain data for drive '$virtdev'\n"
6657                 if !$devinfo->{$devname};
6658
6659             if (defined($options->{storage})) {
6660                 $storeid = $options->{storage} || 'local';
6661             } elsif (!$storeid) {
6662                 $storeid = 'local';
6663             }
6664             $format = 'raw' if !$format;
6665             $devinfo->{$devname}->{devname} = $devname;
6666             $devinfo->{$devname}->{virtdev} = $virtdev;
6667             $devinfo->{$devname}->{format} = $format;
6668             $devinfo->{$devname}->{storeid} = $storeid;
6669
6670             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6671             $check_storage->($storeid, $scfg); # permission and content type check
6672
6673             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6674         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6675             my $virtdev = $1;
6676             my $drive = parse_drive($virtdev, $2);
6677
6678             if (drive_is_cloudinit($drive)) {
6679                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6680                 $storeid = $options->{storage} if defined ($options->{storage});
6681                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6682                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6683
6684                 $check_storage->($storeid, $scfg); # permission and content type check
6685
6686                 $virtdev_hash->{$virtdev} = {
6687                     format => $format,
6688                     storeid => $storeid,
6689                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6690                     is_cloudinit => 1,
6691                 };
6692             }
6693         }
6694     }
6695
6696     return $virtdev_hash;
6697 };
6698
6699 # Helper to allocate and activate all volumes required for a restore
6700 #
6701 # $storecfg: Storage configuration
6702 # $virtdev_hash: as returned by parse_backup_hints()
6703 #
6704 # Returns: { $virtdev => $volid }
6705 my $restore_allocate_devices = sub {
6706     my ($storecfg, $virtdev_hash, $vmid) = @_;
6707
6708     my $map = {};
6709     foreach my $virtdev (sort keys %$virtdev_hash) {
6710         my $d = $virtdev_hash->{$virtdev};
6711         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6712         my $storeid = $d->{storeid};
6713         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6714
6715         # test if requested format is supported
6716         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6717         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6718         $d->{format} = $defFormat if !$supported;
6719
6720         my $name;
6721         if ($d->{is_cloudinit}) {
6722             $name = "vm-$vmid-cloudinit";
6723             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6724             if ($scfg->{path}) {
6725                 $name .= ".$d->{format}";
6726             }
6727         }
6728
6729         my $volid = PVE::Storage::vdisk_alloc(
6730             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6731
6732         print STDERR "new volume ID is '$volid'\n";
6733         $d->{volid} = $volid;
6734
6735         PVE::Storage::activate_volumes($storecfg, [$volid]);
6736
6737         $map->{$virtdev} = $volid;
6738     }
6739
6740     return $map;
6741 };
6742
6743 sub restore_update_config_line {
6744     my ($cookie, $map, $line, $unique) = @_;
6745
6746     return '' if $line =~ m/^\#qmdump\#/;
6747     return '' if $line =~ m/^\#vzdump\#/;
6748     return '' if $line =~ m/^lock:/;
6749     return '' if $line =~ m/^unused\d+:/;
6750     return '' if $line =~ m/^parent:/;
6751
6752     my $res = '';
6753
6754     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6755     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6756         # try to convert old 1.X settings
6757         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6758         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6759             my ($model, $macaddr) = split(/\=/, $devconfig);
6760             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6761             my $net = {
6762                 model => $model,
6763                 bridge => "vmbr$ind",
6764                 macaddr => $macaddr,
6765             };
6766             my $netstr = print_net($net);
6767
6768             $res .= "net$cookie->{netcount}: $netstr\n";
6769             $cookie->{netcount}++;
6770         }
6771     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6772         my ($id, $netstr) = ($1, $2);
6773         my $net = parse_net($netstr);
6774         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6775         $netstr = print_net($net);
6776         $res .= "$id: $netstr\n";
6777     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6778         my $virtdev = $1;
6779         my $value = $3;
6780         my $di = parse_drive($virtdev, $value);
6781         if (defined($di->{backup}) && !$di->{backup}) {
6782             $res .= "#$line";
6783         } elsif ($map->{$virtdev}) {
6784             delete $di->{format}; # format can change on restore
6785             $di->{file} = $map->{$virtdev};
6786             $value = print_drive($di);
6787             $res .= "$virtdev: $value\n";
6788         } else {
6789             $res .= $line;
6790         }
6791     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6792         my $vmgenid = $1;
6793         if ($vmgenid ne '0') {
6794             # always generate a new vmgenid if there was a valid one setup
6795             $vmgenid = generate_uuid();
6796         }
6797         $res .= "vmgenid: $vmgenid\n";
6798     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6799         my ($uuid, $uuid_str);
6800         UUID::generate($uuid);
6801         UUID::unparse($uuid, $uuid_str);
6802         my $smbios1 = parse_smbios1($2);
6803         $smbios1->{uuid} = $uuid_str;
6804         $res .= $1.print_smbios1($smbios1)."\n";
6805     } else {
6806         $res .= $line;
6807     }
6808
6809     return $res;
6810 }
6811
6812 my $restore_deactivate_volumes = sub {
6813     my ($storecfg, $virtdev_hash) = @_;
6814
6815     my $vollist = [];
6816     for my $dev (values $virtdev_hash->%*) {
6817         push $vollist->@*, $dev->{volid} if $dev->{volid};
6818     }
6819
6820     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6821     print STDERR $@ if $@;
6822 };
6823
6824 my $restore_destroy_volumes = sub {
6825     my ($storecfg, $virtdev_hash) = @_;
6826
6827     for my $dev (values $virtdev_hash->%*) {
6828         my $volid = $dev->{volid} or next;
6829         eval {
6830             PVE::Storage::vdisk_free($storecfg, $volid);
6831             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6832         };
6833         print STDERR "unable to cleanup '$volid' - $@" if $@;
6834     }
6835 };
6836
6837 sub restore_merge_config {
6838     my ($filename, $backup_conf_raw, $override_conf) = @_;
6839
6840     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6841     for my $key (keys $override_conf->%*) {
6842         $backup_conf->{$key} = $override_conf->{$key};
6843     }
6844
6845     return $backup_conf;
6846 }
6847
6848 sub scan_volids {
6849     my ($cfg, $vmid) = @_;
6850
6851     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6852
6853     my $volid_hash = {};
6854     foreach my $storeid (keys %$info) {
6855         foreach my $item (@{$info->{$storeid}}) {
6856             next if !($item->{volid} && $item->{size});
6857             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6858             $volid_hash->{$item->{volid}} = $item;
6859         }
6860     }
6861
6862     return $volid_hash;
6863 }
6864
6865 sub update_disk_config {
6866     my ($vmid, $conf, $volid_hash) = @_;
6867
6868     my $changes;
6869     my $prefix = "VM $vmid";
6870
6871     # used and unused disks
6872     my $referenced = {};
6873
6874     # Note: it is allowed to define multiple storages with same path (alias), so
6875     # we need to check both 'volid' and real 'path' (two different volid can point
6876     # to the same path).
6877
6878     my $referencedpath = {};
6879
6880     # update size info
6881     PVE::QemuConfig->foreach_volume($conf, sub {
6882         my ($opt, $drive) = @_;
6883
6884         my $volid = $drive->{file};
6885         return if !$volid;
6886         my $volume = $volid_hash->{$volid};
6887
6888         # mark volid as "in-use" for next step
6889         $referenced->{$volid} = 1;
6890         if ($volume && (my $path = $volume->{path})) {
6891             $referencedpath->{$path} = 1;
6892         }
6893
6894         return if drive_is_cdrom($drive);
6895         return if !$volume;
6896
6897         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6898         if (defined($updated)) {
6899             $changes = 1;
6900             $conf->{$opt} = print_drive($updated);
6901             print "$prefix ($opt): $msg\n";
6902         }
6903     });
6904
6905     # remove 'unusedX' entry if volume is used
6906     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6907         my ($opt, $drive) = @_;
6908
6909         my $volid = $drive->{file};
6910         return if !$volid;
6911
6912         my $path;
6913         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6914         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6915             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6916             $changes = 1;
6917             delete $conf->{$opt};
6918         }
6919
6920         $referenced->{$volid} = 1;
6921         $referencedpath->{$path} = 1 if $path;
6922     });
6923
6924     foreach my $volid (sort keys %$volid_hash) {
6925         next if $volid =~ m/vm-$vmid-state-/;
6926         next if $referenced->{$volid};
6927         my $path = $volid_hash->{$volid}->{path};
6928         next if !$path; # just to be sure
6929         next if $referencedpath->{$path};
6930         $changes = 1;
6931         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6932         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6933         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6934     }
6935
6936     return $changes;
6937 }
6938
6939 sub rescan {
6940     my ($vmid, $nolock, $dryrun) = @_;
6941
6942     my $cfg = PVE::Storage::config();
6943
6944     print "rescan volumes...\n";
6945     my $volid_hash = scan_volids($cfg, $vmid);
6946
6947     my $updatefn =  sub {
6948         my ($vmid) = @_;
6949
6950         my $conf = PVE::QemuConfig->load_config($vmid);
6951
6952         PVE::QemuConfig->check_lock($conf);
6953
6954         my $vm_volids = {};
6955         foreach my $volid (keys %$volid_hash) {
6956             my $info = $volid_hash->{$volid};
6957             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6958         }
6959
6960         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6961
6962         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6963     };
6964
6965     if (defined($vmid)) {
6966         if ($nolock) {
6967             &$updatefn($vmid);
6968         } else {
6969             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6970         }
6971     } else {
6972         my $vmlist = config_list();
6973         foreach my $vmid (keys %$vmlist) {
6974             if ($nolock) {
6975                 &$updatefn($vmid);
6976             } else {
6977                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6978             }
6979         }
6980     }
6981 }
6982
6983 sub restore_proxmox_backup_archive {
6984     my ($archive, $vmid, $user, $options) = @_;
6985
6986     my $storecfg = PVE::Storage::config();
6987
6988     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6989     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6990
6991     my $fingerprint = $scfg->{fingerprint};
6992     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6993
6994     my $repo = PVE::PBSClient::get_repository($scfg);
6995     my $namespace = $scfg->{namespace};
6996
6997     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6998     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6999     local $ENV{PBS_PASSWORD} = $password;
7000     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7001
7002     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7003         PVE::Storage::parse_volname($storecfg, $archive);
7004
7005     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7006
7007     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7008
7009     my $tmpdir = "/var/tmp/vzdumptmp$$";
7010     rmtree $tmpdir;
7011     mkpath $tmpdir;
7012
7013     my $conffile = PVE::QemuConfig->config_file($vmid);
7014      # disable interrupts (always do cleanups)
7015     local $SIG{INT} =
7016         local $SIG{TERM} =
7017         local $SIG{QUIT} =
7018         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7019
7020     # Note: $oldconf is undef if VM does not exists
7021     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7022     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7023     my $new_conf_raw = '';
7024
7025     my $rpcenv = PVE::RPCEnvironment::get();
7026     my $devinfo = {}; # info about drives included in backup
7027     my $virtdev_hash = {}; # info about allocated drives
7028
7029     eval {
7030         # enable interrupts
7031         local $SIG{INT} =
7032             local $SIG{TERM} =
7033             local $SIG{QUIT} =
7034             local $SIG{HUP} =
7035             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7036
7037         my $cfgfn = "$tmpdir/qemu-server.conf";
7038         my $firewall_config_fn = "$tmpdir/fw.conf";
7039         my $index_fn = "$tmpdir/index.json";
7040
7041         my $cmd = "restore";
7042
7043         my $param = [$pbs_backup_name, "index.json", $index_fn];
7044         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7045         my $index = PVE::Tools::file_get_contents($index_fn);
7046         $index = decode_json($index);
7047
7048         foreach my $info (@{$index->{files}}) {
7049             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7050                 my $devname = $1;
7051                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7052                     $devinfo->{$devname}->{size} = $1;
7053                 } else {
7054                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7055                 }
7056             }
7057         }
7058
7059         my $is_qemu_server_backup = scalar(
7060             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7061         );
7062         if (!$is_qemu_server_backup) {
7063             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7064         }
7065         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7066
7067         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7068         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7069
7070         if ($has_firewall_config) {
7071             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7072             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7073
7074             my $pve_firewall_dir = '/etc/pve/firewall';
7075             mkdir $pve_firewall_dir; # make sure the dir exists
7076             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7077         }
7078
7079         my $fh = IO::File->new($cfgfn, "r") ||
7080             die "unable to read qemu-server.conf - $!\n";
7081
7082         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7083
7084         # fixme: rate limit?
7085
7086         # create empty/temp config
7087         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7088
7089         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7090
7091         # allocate volumes
7092         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7093
7094         foreach my $virtdev (sort keys %$virtdev_hash) {
7095             my $d = $virtdev_hash->{$virtdev};
7096             next if $d->{is_cloudinit}; # no need to restore cloudinit
7097
7098             # this fails if storage is unavailable
7099             my $volid = $d->{volid};
7100             my $path = PVE::Storage::path($storecfg, $volid);
7101
7102             # for live-restore we only want to preload the efidisk and TPM state
7103             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7104
7105             my @ns_arg;
7106             if (defined(my $ns = $scfg->{namespace})) {
7107                 @ns_arg = ('--ns', $ns);
7108             }
7109
7110             my $pbs_restore_cmd = [
7111                 '/usr/bin/pbs-restore',
7112                 '--repository', $repo,
7113                 @ns_arg,
7114                 $pbs_backup_name,
7115                 "$d->{devname}.img.fidx",
7116                 $path,
7117                 '--verbose',
7118                 ];
7119
7120             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7121             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7122
7123             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7124                 push @$pbs_restore_cmd, '--skip-zero';
7125             }
7126
7127             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7128             print "restore proxmox backup image: $dbg_cmdstring\n";
7129             run_command($pbs_restore_cmd);
7130         }
7131
7132         $fh->seek(0, 0) || die "seek failed - $!\n";
7133
7134         my $cookie = { netcount => 0 };
7135         while (defined(my $line = <$fh>)) {
7136             $new_conf_raw .= restore_update_config_line(
7137                 $cookie,
7138                 $map,
7139                 $line,
7140                 $options->{unique},
7141             );
7142         }
7143
7144         $fh->close();
7145     };
7146     my $err = $@;
7147
7148     if ($err || !$options->{live}) {
7149         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7150     }
7151
7152     rmtree $tmpdir;
7153
7154     if ($err) {
7155         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7156         die $err;
7157     }
7158
7159     if ($options->{live}) {
7160         # keep lock during live-restore
7161         $new_conf_raw .= "\nlock: create";
7162     }
7163
7164     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7165     check_restore_permissions($rpcenv, $user, $new_conf);
7166     PVE::QemuConfig->write_config($vmid, $new_conf);
7167
7168     eval { rescan($vmid, 1); };
7169     warn $@ if $@;
7170
7171     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7172
7173     if ($options->{live}) {
7174         # enable interrupts
7175         local $SIG{INT} =
7176             local $SIG{TERM} =
7177             local $SIG{QUIT} =
7178             local $SIG{HUP} =
7179             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7180
7181         my $conf = PVE::QemuConfig->load_config($vmid);
7182         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7183
7184         # these special drives are already restored before start
7185         delete $devinfo->{'drive-efidisk0'};
7186         delete $devinfo->{'drive-tpmstate0-backup'};
7187
7188         my $pbs_opts = {
7189             repo => $repo,
7190             keyfile => $keyfile,
7191             snapshot => $pbs_backup_name,
7192             namespace => $namespace,
7193         };
7194         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7195
7196         PVE::QemuConfig->remove_lock($vmid, "create");
7197     }
7198 }
7199
7200 sub pbs_live_restore {
7201     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7202
7203     print "starting VM for live-restore\n";
7204     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7205
7206     my $pbs_backing = {};
7207     for my $ds (keys %$restored_disks) {
7208         $ds =~ m/^drive-(.*)$/;
7209         my $confname = $1;
7210         $pbs_backing->{$confname} = {
7211             repository => $opts->{repo},
7212             snapshot => $opts->{snapshot},
7213             archive => "$ds.img.fidx",
7214         };
7215         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7216         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7217
7218         my $drive = parse_drive($confname, $conf->{$confname});
7219         print "restoring '$ds' to '$drive->{file}'\n";
7220     }
7221
7222     my $drives_streamed = 0;
7223     eval {
7224         # make sure HA doesn't interrupt our restore by stopping the VM
7225         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7226             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7227         }
7228
7229         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7230         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7231         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7232
7233         my $qmeventd_fd = register_qmeventd_handle($vmid);
7234
7235         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7236         # this will effectively collapse the backing image chain consisting of
7237         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7238         # removes itself once all backing images vanish with 'auto-remove=on')
7239         my $jobs = {};
7240         for my $ds (sort keys %$restored_disks) {
7241             my $job_id = "restore-$ds";
7242             mon_cmd($vmid, 'block-stream',
7243                 'job-id' => $job_id,
7244                 device => "$ds",
7245             );
7246             $jobs->{$job_id} = {};
7247         }
7248
7249         mon_cmd($vmid, 'cont');
7250         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7251
7252         print "restore-drive jobs finished successfully, removing all tracking block devices"
7253             ." to disconnect from Proxmox Backup Server\n";
7254
7255         for my $ds (sort keys %$restored_disks) {
7256             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7257         }
7258
7259         close($qmeventd_fd);
7260     };
7261
7262     my $err = $@;
7263
7264     if ($err) {
7265         warn "An error occurred during live-restore: $err\n";
7266         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7267         die "live-restore failed\n";
7268     }
7269 }
7270
7271 sub restore_vma_archive {
7272     my ($archive, $vmid, $user, $opts, $comp) = @_;
7273
7274     my $readfrom = $archive;
7275
7276     my $cfg = PVE::Storage::config();
7277     my $commands = [];
7278     my $bwlimit = $opts->{bwlimit};
7279
7280     my $dbg_cmdstring = '';
7281     my $add_pipe = sub {
7282         my ($cmd) = @_;
7283         push @$commands, $cmd;
7284         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7285         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7286         $readfrom = '-';
7287     };
7288
7289     my $input = undef;
7290     if ($archive eq '-') {
7291         $input = '<&STDIN';
7292     } else {
7293         # If we use a backup from a PVE defined storage we also consider that
7294         # storage's rate limit:
7295         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7296         if (defined($volid)) {
7297             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7298             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7299             if ($readlimit) {
7300                 print STDERR "applying read rate limit: $readlimit\n";
7301                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7302                 $add_pipe->($cstream);
7303             }
7304         }
7305     }
7306
7307     if ($comp) {
7308         my $info = PVE::Storage::decompressor_info('vma', $comp);
7309         my $cmd = $info->{decompressor};
7310         push @$cmd, $readfrom;
7311         $add_pipe->($cmd);
7312     }
7313
7314     my $tmpdir = "/var/tmp/vzdumptmp$$";
7315     rmtree $tmpdir;
7316
7317     # disable interrupts (always do cleanups)
7318     local $SIG{INT} =
7319         local $SIG{TERM} =
7320         local $SIG{QUIT} =
7321         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7322
7323     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7324     POSIX::mkfifo($mapfifo, 0600);
7325     my $fifofh;
7326     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7327
7328     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7329
7330     my $devinfo = {}; # info about drives included in backup
7331     my $virtdev_hash = {}; # info about allocated drives
7332
7333     my $rpcenv = PVE::RPCEnvironment::get();
7334
7335     my $conffile = PVE::QemuConfig->config_file($vmid);
7336
7337     # Note: $oldconf is undef if VM does not exist
7338     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7339     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7340     my $new_conf_raw = '';
7341
7342     my %storage_limits;
7343
7344     my $print_devmap = sub {
7345         my $cfgfn = "$tmpdir/qemu-server.conf";
7346
7347         # we can read the config - that is already extracted
7348         my $fh = IO::File->new($cfgfn, "r") ||
7349             die "unable to read qemu-server.conf - $!\n";
7350
7351         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7352         if (-f $fwcfgfn) {
7353             my $pve_firewall_dir = '/etc/pve/firewall';
7354             mkdir $pve_firewall_dir; # make sure the dir exists
7355             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7356         }
7357
7358         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7359
7360         foreach my $info (values %{$virtdev_hash}) {
7361             my $storeid = $info->{storeid};
7362             next if defined($storage_limits{$storeid});
7363
7364             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7365             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7366             $storage_limits{$storeid} = $limit * 1024;
7367         }
7368
7369         foreach my $devname (keys %$devinfo) {
7370             die "found no device mapping information for device '$devname'\n"
7371                 if !$devinfo->{$devname}->{virtdev};
7372         }
7373
7374         # create empty/temp config
7375         if ($oldconf) {
7376             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7377             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7378         }
7379
7380         # allocate volumes
7381         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7382
7383         # print restore information to $fifofh
7384         foreach my $virtdev (sort keys %$virtdev_hash) {
7385             my $d = $virtdev_hash->{$virtdev};
7386             next if $d->{is_cloudinit}; # no need to restore cloudinit
7387
7388             my $storeid = $d->{storeid};
7389             my $volid = $d->{volid};
7390
7391             my $map_opts = '';
7392             if (my $limit = $storage_limits{$storeid}) {
7393                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7394             }
7395
7396             my $write_zeros = 1;
7397             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7398                 $write_zeros = 0;
7399             }
7400
7401             my $path = PVE::Storage::path($cfg, $volid);
7402
7403             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7404
7405             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7406         }
7407
7408         $fh->seek(0, 0) || die "seek failed - $!\n";
7409
7410         my $cookie = { netcount => 0 };
7411         while (defined(my $line = <$fh>)) {
7412             $new_conf_raw .= restore_update_config_line(
7413                 $cookie,
7414                 $map,
7415                 $line,
7416                 $opts->{unique},
7417             );
7418         }
7419
7420         $fh->close();
7421     };
7422
7423     my $oldtimeout;
7424
7425     eval {
7426         # enable interrupts
7427         local $SIG{INT} =
7428             local $SIG{TERM} =
7429             local $SIG{QUIT} =
7430             local $SIG{HUP} =
7431             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7432         local $SIG{ALRM} = sub { die "got timeout\n"; };
7433
7434         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7435
7436         my $parser = sub {
7437             my $line = shift;
7438
7439             print "$line\n";
7440
7441             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7442                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7443                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7444             } elsif ($line =~ m/^CTIME: /) {
7445                 # we correctly received the vma config, so we can disable
7446                 # the timeout now for disk allocation
7447                 alarm($oldtimeout || 0);
7448                 $oldtimeout = undef;
7449                 &$print_devmap();
7450                 print $fifofh "done\n";
7451                 close($fifofh);
7452                 $fifofh = undef;
7453             }
7454         };
7455
7456         print "restore vma archive: $dbg_cmdstring\n";
7457         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7458     };
7459     my $err = $@;
7460
7461     alarm($oldtimeout) if $oldtimeout;
7462
7463     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7464
7465     close($fifofh) if $fifofh;
7466     unlink $mapfifo;
7467     rmtree $tmpdir;
7468
7469     if ($err) {
7470         $restore_destroy_volumes->($cfg, $virtdev_hash);
7471         die $err;
7472     }
7473
7474     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7475     check_restore_permissions($rpcenv, $user, $new_conf);
7476     PVE::QemuConfig->write_config($vmid, $new_conf);
7477
7478     eval { rescan($vmid, 1); };
7479     warn $@ if $@;
7480
7481     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7482 }
7483
7484 sub restore_tar_archive {
7485     my ($archive, $vmid, $user, $opts) = @_;
7486
7487     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7488         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7489         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7490     }
7491
7492     if ($archive ne '-') {
7493         my $firstfile = tar_archive_read_firstfile($archive);
7494         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7495             if $firstfile ne 'qemu-server.conf';
7496     }
7497
7498     my $storecfg = PVE::Storage::config();
7499
7500     # avoid zombie disks when restoring over an existing VM -> cleanup first
7501     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7502     # skiplock=1 because qmrestore has set the 'create' lock itself already
7503     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7504     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7505
7506     my $tocmd = "/usr/lib/qemu-server/qmextract";
7507
7508     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7509     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7510     $tocmd .= ' --prealloc' if $opts->{prealloc};
7511     $tocmd .= ' --info' if $opts->{info};
7512
7513     # tar option "xf" does not autodetect compression when read from STDIN,
7514     # so we pipe to zcat
7515     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7516         PVE::Tools::shellquote("--to-command=$tocmd");
7517
7518     my $tmpdir = "/var/tmp/vzdumptmp$$";
7519     mkpath $tmpdir;
7520
7521     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7522     local $ENV{VZDUMP_VMID} = $vmid;
7523     local $ENV{VZDUMP_USER} = $user;
7524
7525     my $conffile = PVE::QemuConfig->config_file($vmid);
7526     my $new_conf_raw = '';
7527
7528     # disable interrupts (always do cleanups)
7529     local $SIG{INT} =
7530         local $SIG{TERM} =
7531         local $SIG{QUIT} =
7532         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7533
7534     eval {
7535         # enable interrupts
7536         local $SIG{INT} =
7537             local $SIG{TERM} =
7538             local $SIG{QUIT} =
7539             local $SIG{HUP} =
7540             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7541
7542         if ($archive eq '-') {
7543             print "extracting archive from STDIN\n";
7544             run_command($cmd, input => "<&STDIN");
7545         } else {
7546             print "extracting archive '$archive'\n";
7547             run_command($cmd);
7548         }
7549
7550         return if $opts->{info};
7551
7552         # read new mapping
7553         my $map = {};
7554         my $statfile = "$tmpdir/qmrestore.stat";
7555         if (my $fd = IO::File->new($statfile, "r")) {
7556             while (defined (my $line = <$fd>)) {
7557                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7558                     $map->{$1} = $2 if $1;
7559                 } else {
7560                     print STDERR "unable to parse line in statfile - $line\n";
7561                 }
7562             }
7563             $fd->close();
7564         }
7565
7566         my $confsrc = "$tmpdir/qemu-server.conf";
7567
7568         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7569
7570         my $cookie = { netcount => 0 };
7571         while (defined (my $line = <$srcfd>)) {
7572             $new_conf_raw .= restore_update_config_line(
7573                 $cookie,
7574                 $map,
7575                 $line,
7576                 $opts->{unique},
7577             );
7578         }
7579
7580         $srcfd->close();
7581     };
7582     if (my $err = $@) {
7583         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7584         die $err;
7585     }
7586
7587     rmtree $tmpdir;
7588
7589     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7590
7591     PVE::Cluster::cfs_update(); # make sure we read new file
7592
7593     eval { rescan($vmid, 1); };
7594     warn $@ if $@;
7595 };
7596
7597 sub foreach_storage_used_by_vm {
7598     my ($conf, $func) = @_;
7599
7600     my $sidhash = {};
7601
7602     PVE::QemuConfig->foreach_volume($conf, sub {
7603         my ($ds, $drive) = @_;
7604         return if drive_is_cdrom($drive);
7605
7606         my $volid = $drive->{file};
7607
7608         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7609         $sidhash->{$sid} = $sid if $sid;
7610     });
7611
7612     foreach my $sid (sort keys %$sidhash) {
7613         &$func($sid);
7614     }
7615 }
7616
7617 my $qemu_snap_storage = {
7618     rbd => 1,
7619 };
7620 sub do_snapshots_with_qemu {
7621     my ($storecfg, $volid, $deviceid) = @_;
7622
7623     return if $deviceid =~ m/tpmstate0/;
7624
7625     my $storage_name = PVE::Storage::parse_volume_id($volid);
7626     my $scfg = $storecfg->{ids}->{$storage_name};
7627     die "could not find storage '$storage_name'\n" if !defined($scfg);
7628
7629     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7630         return 1;
7631     }
7632
7633     if ($volid =~ m/\.(qcow2|qed)$/){
7634         return 1;
7635     }
7636
7637     return;
7638 }
7639
7640 sub qga_check_running {
7641     my ($vmid, $nowarn) = @_;
7642
7643     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7644     if ($@) {
7645         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7646         return 0;
7647     }
7648     return 1;
7649 }
7650
7651 sub template_create {
7652     my ($vmid, $conf, $disk) = @_;
7653
7654     my $storecfg = PVE::Storage::config();
7655
7656     PVE::QemuConfig->foreach_volume($conf, sub {
7657         my ($ds, $drive) = @_;
7658
7659         return if drive_is_cdrom($drive);
7660         return if $disk && $ds ne $disk;
7661
7662         my $volid = $drive->{file};
7663         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7664
7665         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7666         $drive->{file} = $voliddst;
7667         $conf->{$ds} = print_drive($drive);
7668         PVE::QemuConfig->write_config($vmid, $conf);
7669     });
7670 }
7671
7672 sub convert_iscsi_path {
7673     my ($path) = @_;
7674
7675     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7676         my $portal = $1;
7677         my $target = $2;
7678         my $lun = $3;
7679
7680         my $initiator_name = get_initiator_name();
7681
7682         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7683                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7684     }
7685
7686     die "cannot convert iscsi path '$path', unkown format\n";
7687 }
7688
7689 sub qemu_img_convert {
7690     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7691
7692     my $storecfg = PVE::Storage::config();
7693     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7694     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7695
7696     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7697
7698     my $cachemode;
7699     my $src_path;
7700     my $src_is_iscsi = 0;
7701     my $src_format;
7702
7703     if ($src_storeid) {
7704         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7705         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7706         $src_format = qemu_img_format($src_scfg, $src_volname);
7707         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7708         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7709         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7710     } elsif (-f $src_volid || -b $src_volid) {
7711         $src_path = $src_volid;
7712         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7713             $src_format = $1;
7714         }
7715     }
7716
7717     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7718
7719     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7720     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7721     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7722     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7723
7724     my $cmd = [];
7725     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7726     push @$cmd, '-l', "snapshot.name=$snapname"
7727         if $snapname && $src_format && $src_format eq "qcow2";
7728     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7729     push @$cmd, '-T', $cachemode if defined($cachemode);
7730     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7731
7732     if ($src_is_iscsi) {
7733         push @$cmd, '--image-opts';
7734         $src_path = convert_iscsi_path($src_path);
7735     } elsif ($src_format) {
7736         push @$cmd, '-f', $src_format;
7737     }
7738
7739     if ($dst_is_iscsi) {
7740         push @$cmd, '--target-image-opts';
7741         $dst_path = convert_iscsi_path($dst_path);
7742     } else {
7743         push @$cmd, '-O', $dst_format;
7744     }
7745
7746     push @$cmd, $src_path;
7747
7748     if (!$dst_is_iscsi && $is_zero_initialized) {
7749         push @$cmd, "zeroinit:$dst_path";
7750     } else {
7751         push @$cmd, $dst_path;
7752     }
7753
7754     my $parser = sub {
7755         my $line = shift;
7756         if($line =~ m/\((\S+)\/100\%\)/){
7757             my $percent = $1;
7758             my $transferred = int($size * $percent / 100);
7759             my $total_h = render_bytes($size, 1);
7760             my $transferred_h = render_bytes($transferred, 1);
7761
7762             print "transferred $transferred_h of $total_h ($percent%)\n";
7763         }
7764
7765     };
7766
7767     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7768     my $err = $@;
7769     die "copy failed: $err" if $err;
7770 }
7771
7772 sub qemu_img_format {
7773     my ($scfg, $volname) = @_;
7774
7775     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7776         return $1;
7777     } else {
7778         return "raw";
7779     }
7780 }
7781
7782 sub qemu_drive_mirror {
7783     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7784
7785     $jobs = {} if !$jobs;
7786
7787     my $qemu_target;
7788     my $format;
7789     $jobs->{"drive-$drive"} = {};
7790
7791     if ($dst_volid =~ /^nbd:/) {
7792         $qemu_target = $dst_volid;
7793         $format = "nbd";
7794     } else {
7795         my $storecfg = PVE::Storage::config();
7796         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7797
7798         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7799
7800         $format = qemu_img_format($dst_scfg, $dst_volname);
7801
7802         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7803
7804         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7805     }
7806
7807     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7808     $opts->{format} = $format if $format;
7809
7810     if (defined($src_bitmap)) {
7811         $opts->{sync} = 'incremental';
7812         $opts->{bitmap} = $src_bitmap;
7813         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7814     }
7815
7816     if (defined($bwlimit)) {
7817         $opts->{speed} = $bwlimit * 1024;
7818         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7819     } else {
7820         print "drive mirror is starting for drive-$drive\n";
7821     }
7822
7823     # if a job already runs for this device we get an error, catch it for cleanup
7824     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7825     if (my $err = $@) {
7826         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7827         warn "$@\n" if $@;
7828         die "mirroring error: $err\n";
7829     }
7830
7831     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7832 }
7833
7834 # $completion can be either
7835 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7836 # 'cancel': wait until all jobs are ready, block-job-cancel them
7837 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7838 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7839 sub qemu_drive_mirror_monitor {
7840     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7841
7842     $completion //= 'complete';
7843     $op //= "mirror";
7844
7845     eval {
7846         my $err_complete = 0;
7847
7848         my $starttime = time ();
7849         while (1) {
7850             die "block job ('$op') timed out\n" if $err_complete > 300;
7851
7852             my $stats = mon_cmd($vmid, "query-block-jobs");
7853             my $ctime = time();
7854
7855             my $running_jobs = {};
7856             for my $stat (@$stats) {
7857                 next if $stat->{type} ne $op;
7858                 $running_jobs->{$stat->{device}} = $stat;
7859             }
7860
7861             my $readycounter = 0;
7862
7863             for my $job_id (sort keys %$jobs) {
7864                 my $job = $running_jobs->{$job_id};
7865
7866                 my $vanished = !defined($job);
7867                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7868                 if($complete || ($vanished && $completion eq 'auto')) {
7869                     print "$job_id: $op-job finished\n";
7870                     delete $jobs->{$job_id};
7871                     next;
7872                 }
7873
7874                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7875
7876                 my $busy = $job->{busy};
7877                 my $ready = $job->{ready};
7878                 if (my $total = $job->{len}) {
7879                     my $transferred = $job->{offset} || 0;
7880                     my $remaining = $total - $transferred;
7881                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7882
7883                     my $duration = $ctime - $starttime;
7884                     my $total_h = render_bytes($total, 1);
7885                     my $transferred_h = render_bytes($transferred, 1);
7886
7887                     my $status = sprintf(
7888                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7889                         render_duration($duration),
7890                     );
7891
7892                     if ($ready) {
7893                         if ($busy) {
7894                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7895                         } else {
7896                             $status .= ", ready";
7897                         }
7898                     }
7899                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7900                     $jobs->{$job_id}->{ready} = $ready;
7901                 }
7902
7903                 $readycounter++ if $job->{ready};
7904             }
7905
7906             last if scalar(keys %$jobs) == 0;
7907
7908             if ($readycounter == scalar(keys %$jobs)) {
7909                 print "all '$op' jobs are ready\n";
7910
7911                 # do the complete later (or has already been done)
7912                 last if $completion eq 'skip' || $completion eq 'auto';
7913
7914                 if ($vmiddst && $vmiddst != $vmid) {
7915                     my $agent_running = $qga && qga_check_running($vmid);
7916                     if ($agent_running) {
7917                         print "freeze filesystem\n";
7918                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7919                         warn $@ if $@;
7920                     } else {
7921                         print "suspend vm\n";
7922                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7923                         warn $@ if $@;
7924                     }
7925
7926                     # if we clone a disk for a new target vm, we don't switch the disk
7927                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7928
7929                     if ($agent_running) {
7930                         print "unfreeze filesystem\n";
7931                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7932                         warn $@ if $@;
7933                     } else {
7934                         print "resume vm\n";
7935                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7936                         warn $@ if $@;
7937                     }
7938
7939                     last;
7940                 } else {
7941
7942                     for my $job_id (sort keys %$jobs) {
7943                         # try to switch the disk if source and destination are on the same guest
7944                         print "$job_id: Completing block job_id...\n";
7945
7946                         my $op;
7947                         if ($completion eq 'complete') {
7948                             $op = 'block-job-complete';
7949                         } elsif ($completion eq 'cancel') {
7950                             $op = 'block-job-cancel';
7951                         } else {
7952                             die "invalid completion value: $completion\n";
7953                         }
7954                         eval { mon_cmd($vmid, $op, device => $job_id) };
7955                         if ($@ =~ m/cannot be completed/) {
7956                             print "$job_id: block job cannot be completed, trying again.\n";
7957                             $err_complete++;
7958                         }else {
7959                             print "$job_id: Completed successfully.\n";
7960                             $jobs->{$job_id}->{complete} = 1;
7961                         }
7962                     }
7963                 }
7964             }
7965             sleep 1;
7966         }
7967     };
7968     my $err = $@;
7969
7970     if ($err) {
7971         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7972         die "block job ($op) error: $err";
7973     }
7974 }
7975
7976 sub qemu_blockjobs_cancel {
7977     my ($vmid, $jobs) = @_;
7978
7979     foreach my $job (keys %$jobs) {
7980         print "$job: Cancelling block job\n";
7981         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7982         $jobs->{$job}->{cancel} = 1;
7983     }
7984
7985     while (1) {
7986         my $stats = mon_cmd($vmid, "query-block-jobs");
7987
7988         my $running_jobs = {};
7989         foreach my $stat (@$stats) {
7990             $running_jobs->{$stat->{device}} = $stat;
7991         }
7992
7993         foreach my $job (keys %$jobs) {
7994
7995             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7996                 print "$job: Done.\n";
7997                 delete $jobs->{$job};
7998             }
7999         }
8000
8001         last if scalar(keys %$jobs) == 0;
8002
8003         sleep 1;
8004     }
8005 }
8006
8007 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8008 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8009 my sub clone_disk_check_io_uring {
8010     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8011
8012     return if !$use_drive_mirror;
8013
8014     # Don't complain when not changing storage.
8015     # Assume if it works for the source, it'll work for the target too.
8016     return if $src_storeid eq $dst_storeid;
8017
8018     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8019     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8020
8021     my $cache_direct = drive_uses_cache_direct($src_drive);
8022
8023     my $src_uses_io_uring;
8024     if ($src_drive->{aio}) {
8025         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8026     } else {
8027         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8028     }
8029
8030     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8031         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8032 }
8033
8034 sub clone_disk {
8035     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8036
8037     my ($vmid, $running) = $source->@{qw(vmid running)};
8038     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8039
8040     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8041     my ($storage, $format) = $dest->@{qw(storage format)};
8042
8043     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8044
8045     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8046         die "cloning from/to EFI disk requires EFI disk\n"
8047             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8048         die "cloning from/to TPM state requires TPM state\n"
8049             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8050
8051         # This would lead to two device nodes in QEMU pointing to the same backing image!
8052         die "cannot change drive name when cloning disk from/to the same VM\n"
8053             if $use_drive_mirror && $vmid == $newvmid;
8054     }
8055
8056     die "cannot move TPM state while VM is running\n"
8057         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8058
8059     my $newvolid;
8060
8061     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8062     print "$src_drivename " if $src_drivename;
8063     print "($drive->{file})\n";
8064
8065     if (!$full) {
8066         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8067         push @$newvollist, $newvolid;
8068     } else {
8069         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8070         my $storeid = $storage || $src_storeid;
8071
8072         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8073
8074         my $name = undef;
8075         my $size = undef;
8076         if (drive_is_cloudinit($drive)) {
8077             $name = "vm-$newvmid-cloudinit";
8078             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8079             if ($scfg->{path}) {
8080                 $name .= ".$dst_format";
8081             }
8082             $snapname = undef;
8083             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8084         } elsif ($dst_drivename eq 'efidisk0') {
8085             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8086         } elsif ($dst_drivename eq 'tpmstate0') {
8087             $dst_format = 'raw';
8088             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8089         } else {
8090             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8091
8092             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8093         }
8094         $newvolid = PVE::Storage::vdisk_alloc(
8095             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8096         );
8097         push @$newvollist, $newvolid;
8098
8099         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8100
8101         if (drive_is_cloudinit($drive)) {
8102             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8103             # if this is the case, we have to complete any block-jobs still there from
8104             # previous drive-mirrors
8105             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8106                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8107             }
8108             goto no_data_clone;
8109         }
8110
8111         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8112         if ($use_drive_mirror) {
8113             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8114                 $completion, $qga, $bwlimit);
8115         } else {
8116             if ($dst_drivename eq 'efidisk0') {
8117                 # the relevant data on the efidisk may be smaller than the source
8118                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8119                 # that is given by the OVMF_VARS.fd
8120                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8121                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8122
8123                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8124
8125                 # better for Ceph if block size is not too small, see bug #3324
8126                 my $bs = 1024*1024;
8127
8128                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8129
8130                 if ($src_format eq 'qcow2' && $snapname) {
8131                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8132                         if !min_version(kvm_user_version(), 6, 2);
8133                     push $cmd->@*, '-l', $snapname;
8134                 }
8135                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8136                 run_command($cmd);
8137             } else {
8138                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8139             }
8140         }
8141     }
8142
8143 no_data_clone:
8144     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8145
8146     my $disk = dclone($drive);
8147     delete $disk->{format};
8148     $disk->{file} = $newvolid;
8149     $disk->{size} = $size if defined($size);
8150
8151     return $disk;
8152 }
8153
8154 sub get_running_qemu_version {
8155     my ($vmid) = @_;
8156     my $res = mon_cmd($vmid, "query-version");
8157     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8158 }
8159
8160 sub qemu_use_old_bios_files {
8161     my ($machine_type) = @_;
8162
8163     return if !$machine_type;
8164
8165     my $use_old_bios_files = undef;
8166
8167     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8168         $machine_type = $1;
8169         $use_old_bios_files = 1;
8170     } else {
8171         my $version = extract_version($machine_type, kvm_user_version());
8172         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8173         # load new efi bios files on migration. So this hack is required to allow
8174         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8175         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8176         $use_old_bios_files = !min_version($version, 2, 4);
8177     }
8178
8179     return ($use_old_bios_files, $machine_type);
8180 }
8181
8182 sub get_efivars_size {
8183     my ($conf, $efidisk) = @_;
8184
8185     my $arch = get_vm_arch($conf);
8186     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8187     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8188     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8189     return -s $ovmf_vars;
8190 }
8191
8192 sub update_efidisk_size {
8193     my ($conf) = @_;
8194
8195     return if !defined($conf->{efidisk0});
8196
8197     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8198     $disk->{size} = get_efivars_size($conf);
8199     $conf->{efidisk0} = print_drive($disk);
8200
8201     return;
8202 }
8203
8204 sub update_tpmstate_size {
8205     my ($conf) = @_;
8206
8207     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8208     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8209     $conf->{tpmstate0} = print_drive($disk);
8210 }
8211
8212 sub create_efidisk($$$$$$$) {
8213     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8214
8215     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8216
8217     my $vars_size_b = -s $ovmf_vars;
8218     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8219     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8220     PVE::Storage::activate_volumes($storecfg, [$volid]);
8221
8222     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8223     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8224
8225     return ($volid, $size/1024);
8226 }
8227
8228 sub vm_iothreads_list {
8229     my ($vmid) = @_;
8230
8231     my $res = mon_cmd($vmid, 'query-iothreads');
8232
8233     my $iothreads = {};
8234     foreach my $iothread (@$res) {
8235         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8236     }
8237
8238     return $iothreads;
8239 }
8240
8241 sub scsihw_infos {
8242     my ($conf, $drive) = @_;
8243
8244     my $maxdev = 0;
8245
8246     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8247         $maxdev = 7;
8248     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8249         $maxdev = 1;
8250     } else {
8251         $maxdev = 256;
8252     }
8253
8254     my $controller = int($drive->{index} / $maxdev);
8255     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8256         ? "virtioscsi"
8257         : "scsihw";
8258
8259     return ($maxdev, $controller, $controller_prefix);
8260 }
8261
8262 sub resolve_dst_disk_format {
8263         my ($storecfg, $storeid, $src_volname, $format) = @_;
8264         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8265
8266         if (!$format) {
8267             # if no target format is specified, use the source disk format as hint
8268             if ($src_volname) {
8269                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8270                 $format = qemu_img_format($scfg, $src_volname);
8271             } else {
8272                 return $defFormat;
8273             }
8274         }
8275
8276         # test if requested format is supported - else use default
8277         my $supported = grep { $_ eq $format } @$validFormats;
8278         $format = $defFormat if !$supported;
8279         return $format;
8280 }
8281
8282 # NOTE: if this logic changes, please update docs & possibly gui logic
8283 sub find_vmstate_storage {
8284     my ($conf, $storecfg) = @_;
8285
8286     # first, return storage from conf if set
8287     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8288
8289     my ($target, $shared, $local);
8290
8291     foreach_storage_used_by_vm($conf, sub {
8292         my ($sid) = @_;
8293         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8294         my $dst = $scfg->{shared} ? \$shared : \$local;
8295         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8296     });
8297
8298     # second, use shared storage where VM has at least one disk
8299     # third, use local storage where VM has at least one disk
8300     # fall back to local storage
8301     $target = $shared // $local // 'local';
8302
8303     return $target;
8304 }
8305
8306 sub generate_uuid {
8307     my ($uuid, $uuid_str);
8308     UUID::generate($uuid);
8309     UUID::unparse($uuid, $uuid_str);
8310     return $uuid_str;
8311 }
8312
8313 sub generate_smbios1_uuid {
8314     return "uuid=".generate_uuid();
8315 }
8316
8317 sub nbd_stop {
8318     my ($vmid) = @_;
8319
8320     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8321 }
8322
8323 sub create_reboot_request {
8324     my ($vmid) = @_;
8325     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8326         or die "failed to create reboot trigger file: $!\n";
8327     close($fh);
8328 }
8329
8330 sub clear_reboot_request {
8331     my ($vmid) = @_;
8332     my $path = "/run/qemu-server/$vmid.reboot";
8333     my $res = 0;
8334
8335     $res = unlink($path);
8336     die "could not remove reboot request for $vmid: $!"
8337         if !$res && $! != POSIX::ENOENT;
8338
8339     return $res;
8340 }
8341
8342 sub bootorder_from_legacy {
8343     my ($conf, $bootcfg) = @_;
8344
8345     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8346     my $bootindex_hash = {};
8347     my $i = 1;
8348     foreach my $o (split(//, $boot)) {
8349         $bootindex_hash->{$o} = $i*100;
8350         $i++;
8351     }
8352
8353     my $bootorder = {};
8354
8355     PVE::QemuConfig->foreach_volume($conf, sub {
8356         my ($ds, $drive) = @_;
8357
8358         if (drive_is_cdrom ($drive, 1)) {
8359             if ($bootindex_hash->{d}) {
8360                 $bootorder->{$ds} = $bootindex_hash->{d};
8361                 $bootindex_hash->{d} += 1;
8362             }
8363         } elsif ($bootindex_hash->{c}) {
8364             $bootorder->{$ds} = $bootindex_hash->{c}
8365                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8366             $bootindex_hash->{c} += 1;
8367         }
8368     });
8369
8370     if ($bootindex_hash->{n}) {
8371         for (my $i = 0; $i < $MAX_NETS; $i++) {
8372             my $netname = "net$i";
8373             next if !$conf->{$netname};
8374             $bootorder->{$netname} = $bootindex_hash->{n};
8375             $bootindex_hash->{n} += 1;
8376         }
8377     }
8378
8379     return $bootorder;
8380 }
8381
8382 # Generate default device list for 'boot: order=' property. Matches legacy
8383 # default boot order, but with explicit device names. This is important, since
8384 # the fallback for when neither 'order' nor the old format is specified relies
8385 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8386 sub get_default_bootdevices {
8387     my ($conf) = @_;
8388
8389     my @ret = ();
8390
8391     # harddisk
8392     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8393     push @ret, $first if $first;
8394
8395     # cdrom
8396     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8397     push @ret, $first if $first;
8398
8399     # network
8400     for (my $i = 0; $i < $MAX_NETS; $i++) {
8401         my $netname = "net$i";
8402         next if !$conf->{$netname};
8403         push @ret, $netname;
8404         last;
8405     }
8406
8407     return \@ret;
8408 }
8409
8410 sub device_bootorder {
8411     my ($conf) = @_;
8412
8413     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8414
8415     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8416
8417     my $bootorder = {};
8418     if (!defined($boot) || $boot->{legacy}) {
8419         $bootorder = bootorder_from_legacy($conf, $boot);
8420     } elsif ($boot->{order}) {
8421         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8422         for my $dev (PVE::Tools::split_list($boot->{order})) {
8423             $bootorder->{$dev} = $i++;
8424         }
8425     }
8426
8427     return $bootorder;
8428 }
8429
8430 sub register_qmeventd_handle {
8431     my ($vmid) = @_;
8432
8433     my $fh;
8434     my $peer = "/var/run/qmeventd.sock";
8435     my $count = 0;
8436
8437     for (;;) {
8438         $count++;
8439         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8440         last if $fh;
8441         if ($! != EINTR && $! != EAGAIN) {
8442             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8443         }
8444         if ($count > 4) {
8445             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8446               . "after $count retries\n";
8447         }
8448         usleep(25000);
8449     }
8450
8451     # send handshake to mark VM as backing up
8452     print $fh to_json({vzdump => {vmid => "$vmid"}});
8453
8454     # return handle to be closed later when inhibit is no longer required
8455     return $fh;
8456 }
8457
8458 # bash completion helper
8459
8460 sub complete_backup_archives {
8461     my ($cmdname, $pname, $cvalue) = @_;
8462
8463     my $cfg = PVE::Storage::config();
8464
8465     my $storeid;
8466
8467     if ($cvalue =~ m/^([^:]+):/) {
8468         $storeid = $1;
8469     }
8470
8471     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8472
8473     my $res = [];
8474     foreach my $id (keys %$data) {
8475         foreach my $item (@{$data->{$id}}) {
8476             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8477             push @$res, $item->{volid} if defined($item->{volid});
8478         }
8479     }
8480
8481     return $res;
8482 }
8483
8484 my $complete_vmid_full = sub {
8485     my ($running) = @_;
8486
8487     my $idlist = vmstatus();
8488
8489     my $res = [];
8490
8491     foreach my $id (keys %$idlist) {
8492         my $d = $idlist->{$id};
8493         if (defined($running)) {
8494             next if $d->{template};
8495             next if $running && $d->{status} ne 'running';
8496             next if !$running && $d->{status} eq 'running';
8497         }
8498         push @$res, $id;
8499
8500     }
8501     return $res;
8502 };
8503
8504 sub complete_vmid {
8505     return &$complete_vmid_full();
8506 }
8507
8508 sub complete_vmid_stopped {
8509     return &$complete_vmid_full(0);
8510 }
8511
8512 sub complete_vmid_running {
8513     return &$complete_vmid_full(1);
8514 }
8515
8516 sub complete_storage {
8517
8518     my $cfg = PVE::Storage::config();
8519     my $ids = $cfg->{ids};
8520
8521     my $res = [];
8522     foreach my $sid (keys %$ids) {
8523         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8524         next if !$ids->{$sid}->{content}->{images};
8525         push @$res, $sid;
8526     }
8527
8528     return $res;
8529 }
8530
8531 sub complete_migration_storage {
8532     my ($cmd, $param, $current_value, $all_args) = @_;
8533
8534     my $targetnode = @$all_args[1];
8535
8536     my $cfg = PVE::Storage::config();
8537     my $ids = $cfg->{ids};
8538
8539     my $res = [];
8540     foreach my $sid (keys %$ids) {
8541         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8542         next if !$ids->{$sid}->{content}->{images};
8543         push @$res, $sid;
8544     }
8545
8546     return $res;
8547 }
8548
8549 sub vm_is_paused {
8550     my ($vmid, $include_suspended) = @_;
8551     my $qmpstatus = eval {
8552         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8553         mon_cmd($vmid, "query-status");
8554     };
8555     warn "$@\n" if $@;
8556     return $qmpstatus && (
8557         $qmpstatus->{status} eq "paused" ||
8558         $qmpstatus->{status} eq "prelaunch" ||
8559         ($include_suspended && $qmpstatus->{status} eq "suspended")
8560     );
8561 }
8562
8563 sub check_volume_storage_type {
8564     my ($storecfg, $vol) = @_;
8565
8566     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8567     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8568     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8569
8570     die "storage '$storeid' does not support content-type '$vtype'\n"
8571         if !$scfg->{content}->{$vtype};
8572
8573     return 1;
8574 }
8575
8576 sub add_nets_bridge_fdb {
8577     my ($conf, $vmid) = @_;
8578
8579     for my $opt (keys %$conf) {
8580         next if $opt !~ m/^net(\d+)$/;
8581         my $iface = "tap${vmid}i$1";
8582         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8583         my $net = parse_net($conf->{$opt}, 1) or next;
8584
8585         my $mac = $net->{macaddr};
8586         if (!$mac) {
8587             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8588                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8589             next;
8590         }
8591
8592         my $bridge = $net->{bridge};
8593         if (!$bridge) {
8594             log_warn("Interface '$iface' not attached to any bridge.");
8595             next;
8596         }
8597         if ($have_sdn) {
8598             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8599         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8600             PVE::Network::add_bridge_fdb($iface, $mac);
8601         }
8602     }
8603 }
8604
8605 sub del_nets_bridge_fdb {
8606     my ($conf, $vmid) = @_;
8607
8608     for my $opt (keys %$conf) {
8609         next if $opt !~ m/^net(\d+)$/;
8610         my $iface = "tap${vmid}i$1";
8611
8612         my $net = parse_net($conf->{$opt}) or next;
8613         my $mac = $net->{macaddr} or next;
8614
8615         my $bridge = $net->{bridge};
8616         if ($have_sdn) {
8617             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8618         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8619             PVE::Network::del_bridge_fdb($iface, $mac);
8620         }
8621     }
8622 }
8623
8624 sub create_ifaces_ipams_ips {
8625     my ($conf, $vmid) = @_;
8626
8627     return if !$have_sdn;
8628
8629     foreach my $opt (keys %$conf) {
8630         if ($opt =~ m/^net(\d+)$/) {
8631             my $value = $conf->{$opt};
8632             my $net = PVE::QemuServer::parse_net($value);
8633             eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8634             warn $@ if $@;
8635         }
8636     }
8637 }
8638
8639 sub delete_ifaces_ipams_ips {
8640     my ($conf, $vmid) = @_;
8641
8642     return if !$have_sdn;
8643
8644     foreach my $opt (keys %$conf) {
8645         if ($opt =~ m/^net(\d+)$/) {
8646             my $net = PVE::QemuServer::parse_net($conf->{$opt});
8647             eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8648             warn $@ if $@;
8649         }
8650     }
8651 }
8652
8653 1;