PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory qw(get_current_memory);
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  62 use PVE::QemuServer::USB;
  63
  64 my $have_sdn;
  65 eval {
  66     require PVE::Network::SDN::Zones;
  67     require PVE::Network::SDN::Vnets;
  68     $have_sdn = 1;
  69 };
  70
  71 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  72 my $OVMF = {
  73     x86_64 => {
  74         '4m-no-smm' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  77         ],
  78         '4m-no-smm-ms' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  81         ],
  82         '4m' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  85         ],
  86         '4m-ms' => [
  87             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  89         ],
  90         # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
  91         # anymore. how can we deperacate this sanely without breaking existing instances, or using
  92         # older backups and snapshot?
  93         default => [
  94             "$EDK2_FW_BASE/OVMF_CODE.fd",
  95             "$EDK2_FW_BASE/OVMF_VARS.fd",
  96         ],
  97     },
  98     aarch64 => {
  99         default => [
 100             "$EDK2_FW_BASE/AAVMF_CODE.fd",
 101             "$EDK2_FW_BASE/AAVMF_VARS.fd",
 102         ],
 103     },
 104 };
 105
 106 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 107
 108 # Note about locking: we use flock on the config file protect against concurent actions.
 109 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 110 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 111 # But you can ignore this kind of lock with the --skiplock flag.
 112
 113 cfs_register_file(
 114     '/qemu-server/',
 115     \&parse_vm_config,
 116     \&write_vm_config
 117 );
 118
 119 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 120     description => "Some command save/restore state from this location.",
 121     type => 'string',
 122     maxLength => 128,
 123     optional => 1,
 124 });
 125
 126 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 127         description => "Specifies the QEMU machine type.",
 128         type => 'string',
 129         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 130         maxLength => 40,
 131         optional => 1,
 132 });
 133
 134 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 135 my $nodename_cache;
 136 sub nodename {
 137     $nodename_cache //= PVE::INotify::nodename();
 138     return $nodename_cache;
 139 }
 140
 141 my $watchdog_fmt = {
 142     model => {
 143         default_key => 1,
 144         type => 'string',
 145         enum => [qw(i6300esb ib700)],
 146         description => "Watchdog type to emulate.",
 147         default => 'i6300esb',
 148         optional => 1,
 149     },
 150     action => {
 151         type => 'string',
 152         enum => [qw(reset shutdown poweroff pause debug none)],
 153         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 154         optional => 1,
 155     },
 156 };
 157 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 158
 159 my $agent_fmt = {
 160     enabled => {
 161         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 162         type => 'boolean',
 163         default => 0,
 164         default_key => 1,
 165     },
 166     fstrim_cloned_disks => {
 167         description => "Run fstrim after moving a disk or migrating the VM.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0,
 171     },
 172     'freeze-fs-on-backup' => {
 173         description => "Freeze/thaw guest filesystems on backup for consistency.",
 174         type => 'boolean',
 175         optional => 1,
 176         default => 1,
 177     },
 178     type => {
 179         description => "Select the agent type",
 180         type => 'string',
 181         default => 'virtio',
 182         optional => 1,
 183         enum => [qw(virtio isa)],
 184     },
 185 };
 186
 187 my $vga_fmt = {
 188     type => {
 189         description => "Select the VGA type.",
 190         type => 'string',
 191         default => 'std',
 192         optional => 1,
 193         default_key => 1,
 194         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 195     },
 196     memory => {
 197         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 198         type => 'integer',
 199         optional => 1,
 200         minimum => 4,
 201         maximum => 512,
 202     },
 203     clipboard => {
 204         description => 'Enable a specific clipboard. If not set, depending on the display type the'
 205             .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
 206         type => 'string',
 207         enum => ['vnc'],
 208         optional => 1,
 209     },
 210 };
 211
 212 my $ivshmem_fmt = {
 213     size => {
 214         type => 'integer',
 215         minimum => 1,
 216         description => "The size of the file in MB.",
 217     },
 218     name => {
 219         type => 'string',
 220         pattern => '[a-zA-Z0-9\-]+',
 221         optional => 1,
 222         format_description => 'string',
 223         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 224     },
 225 };
 226
 227 my $audio_fmt = {
 228     device => {
 229         type => 'string',
 230         enum => [qw(ich9-intel-hda intel-hda AC97)],
 231         description =>  "Configure an audio device."
 232     },
 233     driver =>  {
 234         type => 'string',
 235         enum => ['spice', 'none'],
 236         default => 'spice',
 237         optional => 1,
 238         description => "Driver backend for the audio device."
 239     },
 240 };
 241
 242 my $spice_enhancements_fmt = {
 243     foldersharing => {
 244         type => 'boolean',
 245         optional => 1,
 246         default => '0',
 247         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 248     },
 249     videostreaming =>  {
 250         type => 'string',
 251         enum => ['off', 'all', 'filter'],
 252         default => 'off',
 253         optional => 1,
 254         description => "Enable video streaming. Uses compression for detected video streams."
 255     },
 256 };
 257
 258 my $rng_fmt = {
 259     source => {
 260         type => 'string',
 261         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 262         default_key => 1,
 263         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 264             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 265             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 266             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 267             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 268             ." a hardware RNG from the host.",
 269     },
 270     max_bytes => {
 271         type => 'integer',
 272         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 273             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 274             ." `0` to disable limiting (potentially dangerous!).",
 275         optional => 1,
 276
 277         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 278         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 279         # reading from /dev/urandom
 280         default => 1024,
 281     },
 282     period => {
 283         type => 'integer',
 284         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 285             ." the guest to retrieve another 'max_bytes' of entropy.",
 286         optional => 1,
 287         default => 1000,
 288     },
 289 };
 290
 291 my $meta_info_fmt = {
 292     'ctime' => {
 293         type => 'integer',
 294         description => "The guest creation timestamp as UNIX epoch time",
 295         minimum => 0,
 296         optional => 1,
 297     },
 298     'creation-qemu' => {
 299         type => 'string',
 300         description => "The QEMU (machine) version from the time this VM was created.",
 301         pattern => '\d+(\.\d+)+',
 302         optional => 1,
 303     },
 304 };
 305
 306 my $confdesc = {
 307     onboot => {
 308         optional => 1,
 309         type => 'boolean',
 310         description => "Specifies whether a VM will be started during system bootup.",
 311         default => 0,
 312     },
 313     autostart => {
 314         optional => 1,
 315         type => 'boolean',
 316         description => "Automatic restart after crash (currently ignored).",
 317         default => 0,
 318     },
 319     hotplug => {
 320         optional => 1,
 321         type => 'string', format => 'pve-hotplug-features',
 322         description => "Selectively enable hotplug features. This is a comma separated list of"
 323             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 324             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 325             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 326             ." windows > 7.",
 327         default => 'network,disk,usb',
 328     },
 329     reboot => {
 330         optional => 1,
 331         type => 'boolean',
 332         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 333         default => 1,
 334     },
 335     lock => {
 336         optional => 1,
 337         type => 'string',
 338         description => "Lock/unlock the VM.",
 339         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 340     },
 341     cpulimit => {
 342         optional => 1,
 343         type => 'number',
 344         description => "Limit of CPU usage.",
 345         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 346             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 347         minimum => 0,
 348         maximum => 128,
 349         default => 0,
 350     },
 351     cpuunits => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 355         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 356             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 357             ." weights of all the other running VMs.",
 358         minimum => 1,
 359         maximum => 262144,
 360         default => 'cgroup v1: 1024, cgroup v2: 100',
 361     },
 362     memory => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Memory properties.",
 366         format => $PVE::QemuServer::Memory::memory_fmt
 367     },
 368     balloon => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 372         minimum => 0,
 373     },
 374     shares => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 378             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 379             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 380         minimum => 0,
 381         maximum => 50000,
 382         default => 1000,
 383     },
 384     keyboard => {
 385         optional => 1,
 386         type => 'string',
 387         description => "Keyboard layout for VNC server. This option is generally not required and"
 388         ." is often better handled from within the guest OS.",
 389         enum => PVE::Tools::kvmkeymaplist(),
 390         default => undef,
 391     },
 392     name => {
 393         optional => 1,
 394         type => 'string', format => 'dns-name',
 395         description => "Set a name for the VM. Only used on the configuration web interface.",
 396     },
 397     scsihw => {
 398         optional => 1,
 399         type => 'string',
 400         description => "SCSI controller model",
 401         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 402         default => 'lsi',
 403     },
 404     description => {
 405         optional => 1,
 406         type => 'string',
 407         description => "Description for the VM. Shown in the web-interface VM's summary."
 408             ." This is saved as comment inside the configuration file.",
 409         maxLength => 1024 * 8,
 410     },
 411     ostype => {
 412         optional => 1,
 413         type => 'string',
 414         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 415         description => "Specify guest operating system.",
 416         verbose_description => <<EODESC,
 417 Specify guest operating system. This is used to enable special
 418 optimization/features for specific operating systems:
 419
 420 [horizontal]
 421 other;; unspecified OS
 422 wxp;; Microsoft Windows XP
 423 w2k;; Microsoft Windows 2000
 424 w2k3;; Microsoft Windows 2003
 425 w2k8;; Microsoft Windows 2008
 426 wvista;; Microsoft Windows Vista
 427 win7;; Microsoft Windows 7
 428 win8;; Microsoft Windows 8/2012/2012r2
 429 win10;; Microsoft Windows 10/2016/2019
 430 win11;; Microsoft Windows 11/2022
 431 l24;; Linux 2.4 Kernel
 432 l26;; Linux 2.6 - 6.X Kernel
 433 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 434 EODESC
 435     },
 436     boot => {
 437         optional => 1,
 438         type => 'string', format => 'pve-qm-boot',
 439         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 440             ." key or 'legacy=' is deprecated.",
 441     },
 442     bootdisk => {
 443         optional => 1,
 444         type => 'string', format => 'pve-qm-bootdisk',
 445         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 446         pattern => '(ide|sata|scsi|virtio)\d+',
 447     },
 448     smp => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "The number of CPUs. Please use option -sockets instead.",
 452         minimum => 1,
 453         default => 1,
 454     },
 455     sockets => {
 456         optional => 1,
 457         type => 'integer',
 458         description => "The number of CPU sockets.",
 459         minimum => 1,
 460         default => 1,
 461     },
 462     cores => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "The number of cores per socket.",
 466         minimum => 1,
 467         default => 1,
 468     },
 469     numa => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable NUMA.",
 473         default => 0,
 474     },
 475     hugepages => {
 476         optional => 1,
 477         type => 'string',
 478         description => "Enable/disable hugepages memory.",
 479         enum => [qw(any 2 1024)],
 480     },
 481     keephugepages => {
 482         optional => 1,
 483         type => 'boolean',
 484         default => 0,
 485         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 486             ." after VM shutdown and can be used for subsequent starts.",
 487     },
 488     vcpus => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Number of hotplugged vcpus.",
 492         minimum => 1,
 493         default => 0,
 494     },
 495     acpi => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable ACPI.",
 499         default => 1,
 500     },
 501     agent => {
 502         optional => 1,
 503         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 504         type => 'string',
 505         format => $agent_fmt,
 506     },
 507     kvm => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable KVM hardware virtualization.",
 511         default => 1,
 512     },
 513     tdf => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable time drift fix.",
 517         default => 0,
 518     },
 519     localtime => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 523             ." the `ostype` indicates a Microsoft Windows OS.",
 524     },
 525     freeze => {
 526         optional => 1,
 527         type => 'boolean',
 528         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 529     },
 530     vga => {
 531         optional => 1,
 532         type => 'string', format => $vga_fmt,
 533         description => "Configure the VGA hardware.",
 534         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 535             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 536             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 537             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 538             ." display server. For win* OS you can select how many independent displays you want,"
 539             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 540             ." using a serial device as terminal.",
 541     },
 542     watchdog => {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-watchdog',
 545         description => "Create a virtual hardware watchdog device.",
 546         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 547             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 548             ." else the watchdog will reset the guest (or execute the respective action specified)",
 549     },
 550     startdate => {
 551         optional => 1,
 552         type => 'string',
 553         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 554         description => "Set the initial date of the real time clock. Valid format for date are:"
 555             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 556         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 557         default => 'now',
 558     },
 559     startup =>  get_standard_option('pve-startup-order'),
 560     template => {
 561         optional => 1,
 562         type => 'boolean',
 563         description => "Enable/disable Template.",
 564         default => 0,
 565     },
 566     args => {
 567         optional => 1,
 568         type => 'string',
 569         description => "Arbitrary arguments passed to kvm.",
 570         verbose_description => <<EODESCR,
 571 Arbitrary arguments passed to kvm, for example:
 572
 573 args: -no-reboot -smbios 'type=0,vendor=FOO'
 574
 575 NOTE: this option is for experts only.
 576 EODESCR
 577     },
 578     tablet => {
 579         optional => 1,
 580         type => 'boolean',
 581         default => 1,
 582         description => "Enable/disable the USB tablet device.",
 583         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 584             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 585             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 586             ." may consider disabling this to save some context switches. This is turned off by"
 587             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 588     },
 589     migrate_speed => {
 590         optional => 1,
 591         type => 'integer',
 592         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 593         minimum => 0,
 594         default => 0,
 595     },
 596     migrate_downtime => {
 597         optional => 1,
 598         type => 'number',
 599         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 600         minimum => 0,
 601         default => 0.1,
 602     },
 603     cdrom => {
 604         optional => 1,
 605         type => 'string', format => 'pve-qm-ide',
 606         typetext => '<volume>',
 607         description => "This is an alias for option -ide2",
 608     },
 609     cpu => {
 610         optional => 1,
 611         description => "Emulated CPU type.",
 612         type => 'string',
 613         format => 'pve-vm-cpu-conf',
 614     },
 615     parent => get_standard_option('pve-snapshot-name', {
 616         optional => 1,
 617         description => "Parent snapshot name. This is used internally, and should not be modified.",
 618     }),
 619     snaptime => {
 620         optional => 1,
 621         description => "Timestamp for snapshots.",
 622         type => 'integer',
 623         minimum => 0,
 624     },
 625     vmstate => {
 626         optional => 1,
 627         type => 'string', format => 'pve-volume-id',
 628         description => "Reference to a volume which stores the VM state. This is used internally"
 629             ." for snapshots.",
 630     },
 631     vmstatestorage => get_standard_option('pve-storage-id', {
 632         description => "Default storage for VM state volumes/files.",
 633         optional => 1,
 634     }),
 635     runningmachine => get_standard_option('pve-qemu-machine', {
 636         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 637             ." for snapshots.",
 638     }),
 639     runningcpu => {
 640         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 641             ." internally for snapshots.",
 642         optional => 1,
 643         type => 'string',
 644         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 645         format_description => 'QEMU -cpu parameter'
 646     },
 647     machine => get_standard_option('pve-qemu-machine'),
 648     arch => {
 649         description => "Virtual processor architecture. Defaults to the host.",
 650         optional => 1,
 651         type => 'string',
 652         enum => [qw(x86_64 aarch64)],
 653     },
 654     smbios1 => {
 655         description => "Specify SMBIOS type 1 fields.",
 656         type => 'string', format => 'pve-qm-smbios1',
 657         maxLength => 512,
 658         optional => 1,
 659     },
 660     protection => {
 661         optional => 1,
 662         type => 'boolean',
 663         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 664             ." remove disk operations.",
 665         default => 0,
 666     },
 667     bios => {
 668         optional => 1,
 669         type => 'string',
 670         enum => [ qw(seabios ovmf) ],
 671         description => "Select BIOS implementation.",
 672         default => 'seabios',
 673     },
 674     vmgenid => {
 675         type => 'string',
 676         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 677         format_description => 'UUID',
 678         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 679             ." to disable explicitly.",
 680         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 681             ." value identifier to the guest OS. This allows to notify the guest operating system"
 682             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 683             ." execution or creation from a template). The guest operating system notices the"
 684             ." change, and is then able to react as appropriate by marking its copies of"
 685             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 686             ."Note that auto-creation only works when done through API/CLI create or update methods"
 687             .", but not when manually editing the config file.",
 688         default => "1 (autogenerated)",
 689         optional => 1,
 690     },
 691     hookscript => {
 692         type => 'string',
 693         format => 'pve-volume-id',
 694         optional => 1,
 695         description => "Script that will be executed during various steps in the vms lifetime.",
 696     },
 697     ivshmem => {
 698         type => 'string',
 699         format => $ivshmem_fmt,
 700         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 701             ." the host.",
 702         optional => 1,
 703     },
 704     audio0 => {
 705         type => 'string',
 706         format => $audio_fmt,
 707         description => "Configure a audio device, useful in combination with QXL/Spice.",
 708         optional => 1
 709     },
 710     spice_enhancements => {
 711         type => 'string',
 712         format => $spice_enhancements_fmt,
 713         description => "Configure additional enhancements for SPICE.",
 714         optional => 1
 715     },
 716     tags => {
 717         type => 'string', format => 'pve-tag-list',
 718         description => 'Tags of the VM. This is only meta information.',
 719         optional => 1,
 720     },
 721     rng0 => {
 722         type => 'string',
 723         format => $rng_fmt,
 724         description => "Configure a VirtIO-based Random Number Generator.",
 725         optional => 1,
 726     },
 727     meta => {
 728         type => 'string',
 729         format => $meta_info_fmt,
 730         description => "Some (read-only) meta-information about this guest.",
 731         optional => 1,
 732     },
 733     affinity => {
 734         type => 'string', format => 'pve-cpuset',
 735         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 736         optional => 1,
 737     },
 738 };
 739
 740 my $cicustom_fmt = {
 741     meta => {
 742         type => 'string',
 743         optional => 1,
 744         description => 'Specify a custom file containing all meta data passed to the VM via"
 745             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 746         format => 'pve-volume-id',
 747         format_description => 'volume',
 748     },
 749     network => {
 750         type => 'string',
 751         optional => 1,
 752         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 753         format => 'pve-volume-id',
 754         format_description => 'volume',
 755     },
 756     user => {
 757         type => 'string',
 758         optional => 1,
 759         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 760         format => 'pve-volume-id',
 761         format_description => 'volume',
 762     },
 763     vendor => {
 764         type => 'string',
 765         optional => 1,
 766         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 767         format => 'pve-volume-id',
 768         format_description => 'volume',
 769     },
 770 };
 771 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 772
 773 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 774 my $confdesc_cloudinit = {
 775     citype => {
 776         optional => 1,
 777         type => 'string',
 778         description => 'Specifies the cloud-init configuration format. The default depends on the'
 779             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 780             .' and `configdrive2` for windows.',
 781         enum => ['configdrive2', 'nocloud', 'opennebula'],
 782     },
 783     ciuser => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: User name to change ssh keys and password for instead of the"
 787             ." image's configured default user.",
 788     },
 789     cipassword => {
 790         optional => 1,
 791         type => 'string',
 792         description => 'cloud-init: Password to assign the user. Using this is generally not'
 793             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 794             .' support hashed passwords.',
 795     },
 796     ciupgrade => {
 797         optional => 1,
 798         type => 'boolean',
 799         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 800         default => 1,
 801     },
 802     cicustom => {
 803         optional => 1,
 804         type => 'string',
 805         description => 'cloud-init: Specify custom files to replace the automatically generated'
 806             .' ones at start.',
 807         format => 'pve-qm-cicustom',
 808     },
 809     searchdomain => {
 810         optional => 1,
 811         type => 'string',
 812         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 813             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 814             .' are set.',
 815     },
 816     nameserver => {
 817         optional => 1,
 818         type => 'string', format => 'address-list',
 819         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 820             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 821             .' are set.',
 822     },
 823     sshkeys => {
 824         optional => 1,
 825         type => 'string',
 826         format => 'urlencoded',
 827         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 828     },
 829 };
 830
 831 # what about other qemu settings ?
 832 #cpu => 'string',
 833 #machine => 'string',
 834 #fda => 'file',
 835 #fdb => 'file',
 836 #mtdblock => 'file',
 837 #sd => 'file',
 838 #pflash => 'file',
 839 #snapshot => 'bool',
 840 #bootp => 'file',
 841 ##tftp => 'dir',
 842 ##smb => 'dir',
 843 #kernel => 'file',
 844 #append => 'string',
 845 #initrd => 'file',
 846 ##soundhw => 'string',
 847
 848 while (my ($k, $v) = each %$confdesc) {
 849     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 850 }
 851
 852 my $MAX_NETS = 32;
 853 my $MAX_SERIAL_PORTS = 4;
 854 my $MAX_PARALLEL_PORTS = 3;
 855
 856 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 857     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 858 }
 859
 860 my $nic_model_list = [
 861     'e1000',
 862     'e1000-82540em',
 863     'e1000-82544gc',
 864     'e1000-82545em',
 865     'e1000e',
 866     'i82551',
 867     'i82557b',
 868     'i82559er',
 869     'ne2k_isa',
 870     'ne2k_pci',
 871     'pcnet',
 872     'rtl8139',
 873     'virtio',
 874     'vmxnet3',
 875 ];
 876 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 877
 878 my $net_fmt_bridge_descr = <<__EOD__;
 879 Bridge to attach the network device to. The Proxmox VE standard bridge
 880 is called 'vmbr0'.
 881
 882 If you do not specify a bridge, we create a kvm user (NATed) network
 883 device, which provides DHCP and DNS services. The following addresses
 884 are used:
 885
 886  10.0.2.2   Gateway
 887  10.0.2.3   DNS Server
 888  10.0.2.4   SMB Server
 889
 890 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 891 __EOD__
 892
 893 my $net_fmt = {
 894     macaddr  => get_standard_option('mac-addr', {
 895         description => "MAC address. That address must be unique withing your network. This is"
 896             ." automatically generated if not specified.",
 897     }),
 898     model => {
 899         type => 'string',
 900         description => "Network Card Model. The 'virtio' model provides the best performance with"
 901             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 902             ." best to use 'e1000'.",
 903         enum => $nic_model_list,
 904         default_key => 1,
 905     },
 906     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 907     bridge => get_standard_option('pve-bridge-id', {
 908         description => $net_fmt_bridge_descr,
 909         optional => 1,
 910     }),
 911     queues => {
 912         type => 'integer',
 913         minimum => 0, maximum => 64,
 914         description => 'Number of packet queues to be used on the device.',
 915         optional => 1,
 916     },
 917     rate => {
 918         type => 'number',
 919         minimum => 0,
 920         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 921         optional => 1,
 922     },
 923     tag => {
 924         type => 'integer',
 925         minimum => 1, maximum => 4094,
 926         description => 'VLAN tag to apply to packets on this interface.',
 927         optional => 1,
 928     },
 929     trunks => {
 930         type => 'string',
 931         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 932         description => 'VLAN trunks to pass through this interface.',
 933         format_description => 'vlanid[;vlanid...]',
 934         optional => 1,
 935     },
 936     firewall => {
 937         type => 'boolean',
 938         description => 'Whether this interface should be protected by the firewall.',
 939         optional => 1,
 940     },
 941     link_down => {
 942         type => 'boolean',
 943         description => 'Whether this interface should be disconnected (like pulling the plug).',
 944         optional => 1,
 945     },
 946     mtu => {
 947         type => 'integer',
 948         minimum => 1, maximum => 65520,
 949         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 950         optional => 1,
 951     },
 952 };
 953
 954 my $netdesc = {
 955     optional => 1,
 956     type => 'string', format => $net_fmt,
 957     description => "Specify network devices.",
 958 };
 959
 960 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 961
 962 my $ipconfig_fmt = {
 963     ip => {
 964         type => 'string',
 965         format => 'pve-ipv4-config',
 966         format_description => 'IPv4Format/CIDR',
 967         description => 'IPv4 address in CIDR format.',
 968         optional => 1,
 969         default => 'dhcp',
 970     },
 971     gw => {
 972         type => 'string',
 973         format => 'ipv4',
 974         format_description => 'GatewayIPv4',
 975         description => 'Default gateway for IPv4 traffic.',
 976         optional => 1,
 977         requires => 'ip',
 978     },
 979     ip6 => {
 980         type => 'string',
 981         format => 'pve-ipv6-config',
 982         format_description => 'IPv6Format/CIDR',
 983         description => 'IPv6 address in CIDR format.',
 984         optional => 1,
 985         default => 'dhcp',
 986     },
 987     gw6 => {
 988         type => 'string',
 989         format => 'ipv6',
 990         format_description => 'GatewayIPv6',
 991         description => 'Default gateway for IPv6 traffic.',
 992         optional => 1,
 993         requires => 'ip6',
 994     },
 995 };
 996 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 997 my $ipconfigdesc = {
 998     optional => 1,
 999     type => 'string', format => 'pve-qm-ipconfig',
1000     description => <<'EODESCR',
1001 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1002
1003 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1004
1005 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1006 gateway should be provided.
1007 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1008 cloud-init 19.4 or newer.
1009
1010 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1011 dhcp on IPv4.
1012 EODESCR
1013 };
1014 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1015
1016 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1017     $confdesc->{"net$i"} = $netdesc;
1018     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1019 }
1020
1021 foreach my $key (keys %$confdesc_cloudinit) {
1022     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1023 }
1024
1025 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1026 sub pve_verify_cpuset {
1027     my ($set_text, $noerr) = @_;
1028
1029     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1030
1031     if ($@) {
1032         return if $noerr;
1033         die "unable to parse cpuset option\n";
1034     }
1035
1036     return PVE::CpuSet->new($members)->short_string();
1037 }
1038
1039 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1040 sub verify_volume_id_or_qm_path {
1041     my ($volid, $noerr) = @_;
1042
1043     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1044
1045     return verify_volume_id_or_absolute_path($volid, $noerr);
1046 }
1047
1048 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1049 sub verify_volume_id_or_absolute_path {
1050     my ($volid, $noerr) = @_;
1051
1052     return $volid if $volid =~ m|^/|;
1053
1054     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1055     if ($@) {
1056         return if $noerr;
1057         die $@;
1058     }
1059     return $volid;
1060 }
1061
1062 my $serialdesc = {
1063         optional => 1,
1064         type => 'string',
1065         pattern => '(/dev/.+|socket)',
1066         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1067         verbose_description =>  <<EODESCR,
1068 Create a serial device inside the VM (n is 0 to 3), and pass through a
1069 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1070 host side (use 'qm terminal' to open a terminal connection).
1071
1072 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1073 use with special care.
1074
1075 CAUTION: Experimental! User reported problems with this option.
1076 EODESCR
1077 };
1078
1079 my $paralleldesc= {
1080         optional => 1,
1081         type => 'string',
1082         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1083         description =>  "Map host parallel devices (n is 0 to 2).",
1084         verbose_description =>  <<EODESCR,
1085 Map host parallel devices (n is 0 to 2).
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1095     $confdesc->{"parallel$i"} = $paralleldesc;
1096 }
1097
1098 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1099     $confdesc->{"serial$i"} = $serialdesc;
1100 }
1101
1102 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1103     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1104 }
1105
1106 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1107     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1108 }
1109
1110 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1111     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1112 }
1113
1114 my $boot_fmt = {
1115     legacy => {
1116         optional => 1,
1117         default_key => 1,
1118         type => 'string',
1119         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1120                      . " Deprecated, use 'order=' instead.",
1121         pattern => '[acdn]{1,4}',
1122         format_description => "[acdn]{1,4}",
1123
1124         # note: this is also the fallback if boot: is not given at all
1125         default => 'cdn',
1126     },
1127     order => {
1128         optional => 1,
1129         type => 'string',
1130         format => 'pve-qm-bootdev-list',
1131         format_description => "device[;device...]",
1132         description => <<EODESC,
1133 The guest will attempt to boot from devices in the order they appear here.
1134
1135 Disks, optical drives and passed-through storage USB devices will be directly
1136 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1137 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1138
1139 Note that only devices in this list will be marked as bootable and thus loaded
1140 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1141 (e.g. software-raid), you need to specify all of them here.
1142
1143 Overrides the deprecated 'legacy=[acdn]*' value when given.
1144 EODESC
1145     },
1146 };
1147 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1148
1149 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1150 sub verify_bootdev {
1151     my ($dev, $noerr) = @_;
1152
1153     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1154     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1155
1156     my $check = sub {
1157         my ($base) = @_;
1158         return 0 if $dev !~ m/^$base\d+$/;
1159         return 0 if !$confdesc->{$dev};
1160         return 1;
1161     };
1162
1163     return $dev if $check->("net");
1164     return $dev if $check->("usb");
1165     return $dev if $check->("hostpci");
1166
1167     return if $noerr;
1168     die "invalid boot device '$dev'\n";
1169 }
1170
1171 sub print_bootorder {
1172     my ($devs) = @_;
1173     return "" if !@$devs;
1174     my $data = { order => join(';', @$devs) };
1175     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1176 }
1177
1178 my $kvm_api_version = 0;
1179
1180 sub kvm_version {
1181     return $kvm_api_version if $kvm_api_version;
1182
1183     open my $fh, '<', '/dev/kvm' or return;
1184
1185     # 0xae00 => KVM_GET_API_VERSION
1186     $kvm_api_version = ioctl($fh, 0xae00, 0);
1187     close($fh);
1188
1189     return $kvm_api_version;
1190 }
1191
1192 my $kvm_user_version = {};
1193 my $kvm_mtime = {};
1194
1195 sub kvm_user_version {
1196     my ($binary) = @_;
1197
1198     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1199     my $st = stat($binary);
1200
1201     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1202     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1203         $cachedmtime == $st->mtime;
1204
1205     $kvm_user_version->{$binary} = 'unknown';
1206     $kvm_mtime->{$binary} = $st->mtime;
1207
1208     my $code = sub {
1209         my $line = shift;
1210         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1211             $kvm_user_version->{$binary} = $2;
1212         }
1213     };
1214
1215     eval { run_command([$binary, '--version'], outfunc => $code); };
1216     warn $@ if $@;
1217
1218     return $kvm_user_version->{$binary};
1219
1220 }
1221 my sub extract_version {
1222     my ($machine_type, $version) = @_;
1223     $version = kvm_user_version() if !defined($version);
1224     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1225 }
1226
1227 sub kernel_has_vhost_net {
1228     return -c '/dev/vhost-net';
1229 }
1230
1231 sub option_exists {
1232     my $key = shift;
1233     return defined($confdesc->{$key});
1234 }
1235
1236 my $cdrom_path;
1237 sub get_cdrom_path {
1238
1239     return $cdrom_path if defined($cdrom_path);
1240
1241     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1242
1243     if (!defined($cdrom_path)) {
1244         log_warn("no physical CD-ROM available, ignoring");
1245         $cdrom_path = '';
1246     }
1247
1248     return $cdrom_path;
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269      if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1314             if !$vtype;
1315         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1316         verify_media_type($opt, $vtype, $drive->{media});
1317         $drive->{file} = $volid;
1318     }
1319
1320     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1321 }
1322
1323 sub parse_hotplug_features {
1324     my ($data) = @_;
1325
1326     my $res = {};
1327
1328     return $res if $data eq '0';
1329
1330     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1331
1332     foreach my $feature (PVE::Tools::split_list($data)) {
1333         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1334             $res->{$1} = 1;
1335         } else {
1336             die "invalid hotplug feature '$feature'\n";
1337         }
1338     }
1339     return $res;
1340 }
1341
1342 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1343 sub pve_verify_hotplug_features {
1344     my ($value, $noerr) = @_;
1345
1346     return $value if parse_hotplug_features($value);
1347
1348     return if $noerr;
1349
1350     die "unable to parse hotplug option\n";
1351 }
1352
1353 sub assert_clipboard_config {
1354     my ($vga) = @_;
1355
1356     my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1357
1358     if (
1359         $vga->{'clipboard'}
1360         && $vga->{'clipboard'} eq 'vnc'
1361         && $vga->{type}
1362         && $vga->{type} !~ $clipboard_regex
1363     ) {
1364         die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1365     }
1366 }
1367
1368 sub print_tabletdevice_full {
1369     my ($conf, $arch) = @_;
1370
1371     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1372
1373     # we use uhci for old VMs because tablet driver was buggy in older qemu
1374     my $usbbus;
1375     if ($q35 || $arch eq 'aarch64') {
1376         $usbbus = 'ehci';
1377     } else {
1378         $usbbus = 'uhci';
1379     }
1380
1381     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1382 }
1383
1384 sub print_keyboarddevice_full {
1385     my ($conf, $arch) = @_;
1386
1387     return if $arch ne 'aarch64';
1388
1389     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1390 }
1391
1392 my sub get_drive_id {
1393     my ($drive) = @_;
1394     return "$drive->{interface}$drive->{index}";
1395 }
1396
1397 sub print_drivedevice_full {
1398     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1399
1400     my $device = '';
1401     my $maxdev = 0;
1402
1403     my $drive_id = get_drive_id($drive);
1404     if ($drive->{interface} eq 'virtio') {
1405         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1406         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1407         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1408     } elsif ($drive->{interface} eq 'scsi') {
1409
1410         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1411         my $unit = $drive->{index} % $maxdev;
1412
1413         my $machine_version = extract_version($machine_type, kvm_user_version());
1414         my $devicetype  = PVE::QemuServer::Drive::get_scsi_devicetype(
1415             $drive, $storecfg, $machine_version);
1416
1417         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1418             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1419         } else {
1420             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1421                 .",lun=$drive->{index}";
1422         }
1423         $device .= ",drive=drive-$drive_id,id=$drive_id";
1424
1425         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1426             $device .= ",rotation_rate=1";
1427         }
1428         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1429
1430     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1431         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1432         my $controller = int($drive->{index} / $maxdev);
1433         my $unit = $drive->{index} % $maxdev;
1434
1435         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1436         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1437         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1438         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1439         # were before. Move odd ones up by 2 where they don't clash.
1440         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1441             $controller += 2 * ($unit % 2);
1442             $unit = 0;
1443         }
1444
1445         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1446
1447         $device = "ide-$devicetype";
1448         if ($drive->{interface} eq 'ide') {
1449             $device .= ",bus=ide.$controller,unit=$unit";
1450         } else {
1451             $device .= ",bus=ahci$controller.$unit";
1452         }
1453         $device .= ",drive=drive-$drive_id,id=$drive_id";
1454
1455         if ($devicetype eq 'hd') {
1456             if (my $model = $drive->{model}) {
1457                 $model = URI::Escape::uri_unescape($model);
1458                 $device .= ",model=$model";
1459             }
1460             if ($drive->{ssd}) {
1461                 $device .= ",rotation_rate=1";
1462             }
1463         }
1464         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1465     } elsif ($drive->{interface} eq 'usb') {
1466         die "implement me";
1467         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1468     } else {
1469         die "unsupported interface type";
1470     }
1471
1472     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1473
1474     if (my $serial = $drive->{serial}) {
1475         $serial = URI::Escape::uri_unescape($serial);
1476         $device .= ",serial=$serial";
1477     }
1478
1479
1480     return $device;
1481 }
1482
1483 sub get_initiator_name {
1484     my $initiator;
1485
1486     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1487     while (defined(my $line = <$fh>)) {
1488         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1489         $initiator = $1;
1490         last;
1491     }
1492     $fh->close();
1493
1494     return $initiator;
1495 }
1496
1497 my sub storage_allows_io_uring_default {
1498     my ($scfg, $cache_direct) = @_;
1499
1500     # io_uring with cache mode writeback or writethrough on krbd will hang...
1501     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1502
1503     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1504     # sometimes, just plain disable...
1505     return if $scfg && $scfg->{type} eq 'lvm';
1506
1507     # io_uring causes problems when used with CIFS since kernel 5.15
1508     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1509     return if $scfg && $scfg->{type} eq 'cifs';
1510
1511     return 1;
1512 }
1513
1514 my sub drive_uses_cache_direct {
1515     my ($drive, $scfg) = @_;
1516
1517     my $cache_direct = 0;
1518
1519     if (my $cache = $drive->{cache}) {
1520         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1521     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1522         $cache_direct = 1;
1523     }
1524
1525     return $cache_direct;
1526 }
1527
1528 sub print_drive_commandline_full {
1529     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1530
1531     my $path;
1532     my $volid = $drive->{file};
1533     my $format = $drive->{format};
1534     my $drive_id = get_drive_id($drive);
1535
1536     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1537     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1538
1539     if (drive_is_cdrom($drive)) {
1540         $path = get_iso_path($storecfg, $vmid, $volid);
1541         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1542     } else {
1543         if ($storeid) {
1544             $path = PVE::Storage::path($storecfg, $volid);
1545             $format //= qemu_img_format($scfg, $volname);
1546         } else {
1547             $path = $volid;
1548             $format //= "raw";
1549         }
1550    }
1551
1552    my $is_rbd = $path =~ m/^rbd:/;
1553
1554     my $opts = '';
1555     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1556     foreach my $o (@qemu_drive_options) {
1557         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1558     }
1559
1560     # snapshot only accepts on|off
1561     if (defined($drive->{snapshot})) {
1562         my $v = $drive->{snapshot} ? 'on' : 'off';
1563         $opts .= ",snapshot=$v";
1564     }
1565
1566     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1567         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1568     }
1569
1570     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1571         my ($dir, $qmpname) = @$type;
1572         if (my $v = $drive->{"mbps$dir"}) {
1573             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1574         }
1575         if (my $v = $drive->{"mbps${dir}_max"}) {
1576             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1577         }
1578         if (my $v = $drive->{"bps${dir}_max_length"}) {
1579             $opts .= ",throttling.bps$qmpname-max-length=$v";
1580         }
1581         if (my $v = $drive->{"iops${dir}"}) {
1582             $opts .= ",throttling.iops$qmpname=$v";
1583         }
1584         if (my $v = $drive->{"iops${dir}_max"}) {
1585             $opts .= ",throttling.iops$qmpname-max=$v";
1586         }
1587         if (my $v = $drive->{"iops${dir}_max_length"}) {
1588             $opts .= ",throttling.iops$qmpname-max-length=$v";
1589         }
1590     }
1591
1592     if ($pbs_name) {
1593         $format = "rbd" if $is_rbd;
1594         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1595             if !$format;
1596         $opts .= ",format=alloc-track,file.driver=$format";
1597     } elsif ($format) {
1598         $opts .= ",format=$format";
1599     }
1600
1601     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1602
1603     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1604
1605     if (!$drive->{aio}) {
1606         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1607             # io_uring supports all cache modes
1608             $opts .= ",aio=io_uring";
1609         } else {
1610             # aio native works only with O_DIRECT
1611             if($cache_direct) {
1612                 $opts .= ",aio=native";
1613             } else {
1614                 $opts .= ",aio=threads";
1615             }
1616         }
1617     }
1618
1619     if (!drive_is_cdrom($drive)) {
1620         my $detectzeroes;
1621         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1622             $detectzeroes = 'off';
1623         } elsif ($drive->{discard}) {
1624             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1625         } else {
1626             # This used to be our default with discard not being specified:
1627             $detectzeroes = 'on';
1628         }
1629
1630         # note: 'detect-zeroes' works per blockdev and we want it to persist
1631         # after the alloc-track is removed, so put it on 'file' directly
1632         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1633         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1634     }
1635
1636     if ($pbs_name) {
1637         $opts .= ",backing=$pbs_name";
1638         $opts .= ",auto-remove=on";
1639     }
1640
1641     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1642     my $file_param = "file";
1643     if ($pbs_name) {
1644         # non-rbd drivers require the underlying file to be a seperate block
1645         # node, so add a second .file indirection
1646         $file_param .= ".file" if !$is_rbd;
1647         $file_param .= ".filename";
1648     }
1649     my $pathinfo = $path ? "$file_param=$path," : '';
1650
1651     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1652 }
1653
1654 sub print_pbs_blockdev {
1655     my ($pbs_conf, $pbs_name) = @_;
1656     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1657     $blockdev .= ",repository=$pbs_conf->{repository}";
1658     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1659     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1660     $blockdev .= ",archive=$pbs_conf->{archive}";
1661     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1662     return $blockdev;
1663 }
1664
1665 sub print_netdevice_full {
1666     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1667
1668     my $device = $net->{model};
1669     if ($net->{model} eq 'virtio') {
1670          $device = 'virtio-net-pci';
1671      };
1672
1673     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1674     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1675     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1676         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1677         # and out of each queue plus one config interrupt and control vector queue
1678         my $vectors = $net->{queues} * 2 + 2;
1679         $tmpstr .= ",vectors=$vectors,mq=on";
1680         if (min_version($machine_version, 7, 1)) {
1681             $tmpstr .= ",packed=on";
1682         }
1683     }
1684
1685     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1686         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1687     }
1688
1689     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1690
1691     if (my $mtu = $net->{mtu}) {
1692         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1693             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1694             if ($mtu == 1) {
1695                  $mtu = $bridge_mtu;
1696             } elsif ($mtu < 576) {
1697                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1698             } elsif ($mtu > $bridge_mtu) {
1699                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1700             }
1701             $tmpstr .= ",host_mtu=$mtu";
1702         } else {
1703             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1704         }
1705     }
1706
1707     if ($use_old_bios_files) {
1708         my $romfile;
1709         if ($device eq 'virtio-net-pci') {
1710             $romfile = 'pxe-virtio.rom';
1711         } elsif ($device eq 'e1000') {
1712             $romfile = 'pxe-e1000.rom';
1713         } elsif ($device eq 'e1000e') {
1714             $romfile = 'pxe-e1000e.rom';
1715         } elsif ($device eq 'ne2k') {
1716             $romfile = 'pxe-ne2k_pci.rom';
1717         } elsif ($device eq 'pcnet') {
1718             $romfile = 'pxe-pcnet.rom';
1719         } elsif ($device eq 'rtl8139') {
1720             $romfile = 'pxe-rtl8139.rom';
1721         }
1722         $tmpstr .= ",romfile=$romfile" if $romfile;
1723     }
1724
1725     return $tmpstr;
1726 }
1727
1728 sub print_netdev_full {
1729     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1730
1731     my $i = '';
1732     if ($netid =~ m/^net(\d+)$/) {
1733         $i = int($1);
1734     }
1735
1736     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1737
1738     my $ifname = "tap${vmid}i$i";
1739
1740     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1741     die "interface name '$ifname' is too long (max 15 character)\n"
1742         if length($ifname) >= 16;
1743
1744     my $vhostparam = '';
1745     if (is_native($arch)) {
1746         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1747     }
1748
1749     my $vmname = $conf->{name} || "vm$vmid";
1750
1751     my $netdev = "";
1752     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1753
1754     if ($net->{bridge}) {
1755         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1756             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1757     } else {
1758         $netdev = "type=user,id=$netid,hostname=$vmname";
1759     }
1760
1761     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1762
1763     return $netdev;
1764 }
1765
1766 my $vga_map = {
1767     'cirrus' => 'cirrus-vga',
1768     'std' => 'VGA',
1769     'vmware' => 'vmware-svga',
1770     'virtio' => 'virtio-vga',
1771     'virtio-gl' => 'virtio-vga-gl',
1772 };
1773
1774 sub print_vga_device {
1775     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1776
1777     my $type = $vga_map->{$vga->{type}};
1778     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1779         $type = 'virtio-gpu';
1780     }
1781     my $vgamem_mb = $vga->{memory};
1782
1783     my $max_outputs = '';
1784     if ($qxlnum) {
1785         $type = $id ? 'qxl' : 'qxl-vga';
1786
1787         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1788             # set max outputs so linux can have up to 4 qxl displays with one device
1789             if (min_version($machine_version, 4, 1)) {
1790                 $max_outputs = ",max_outputs=4";
1791             }
1792         }
1793     }
1794
1795     die "no devicetype for $vga->{type}\n" if !$type;
1796
1797     my $memory = "";
1798     if ($vgamem_mb) {
1799         if ($vga->{type} =~ /^virtio/) {
1800             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1801             $memory = ",max_hostmem=$bytes";
1802         } elsif ($qxlnum) {
1803             # from https://www.spice-space.org/multiple-monitors.html
1804             $memory = ",vgamem_mb=$vga->{memory}";
1805             my $ram = $vgamem_mb * 4;
1806             my $vram = $vgamem_mb * 2;
1807             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1808         } else {
1809             $memory = ",vgamem_mb=$vga->{memory}";
1810         }
1811     } elsif ($qxlnum && $id) {
1812         $memory = ",ram_size=67108864,vram_size=33554432";
1813     }
1814
1815     my $edidoff = "";
1816     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1817         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1818     }
1819
1820     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1821     my $vgaid = "vga" . ($id // '');
1822     my $pciaddr;
1823     if ($q35 && $vgaid eq 'vga') {
1824         # the first display uses pcie.0 bus on q35 machines
1825         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1826     } else {
1827         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1828     }
1829
1830     if ($vga->{type} eq 'virtio-gl') {
1831         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1832         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1833             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1834
1835         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1836             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1837     }
1838
1839     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1840 }
1841
1842 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1843 sub parse_net {
1844     my ($data, $disable_mac_autogen) = @_;
1845
1846     my $res = eval { parse_property_string($net_fmt, $data) };
1847     if ($@) {
1848         warn $@;
1849         return;
1850     }
1851     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1852         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1853         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1854     }
1855     return $res;
1856 }
1857
1858 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1859 sub parse_ipconfig {
1860     my ($data) = @_;
1861
1862     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1863     if ($@) {
1864         warn $@;
1865         return;
1866     }
1867
1868     if ($res->{gw} && !$res->{ip}) {
1869         warn 'gateway specified without specifying an IP address';
1870         return;
1871     }
1872     if ($res->{gw6} && !$res->{ip6}) {
1873         warn 'IPv6 gateway specified without specifying an IPv6 address';
1874         return;
1875     }
1876     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1877         warn 'gateway specified together with DHCP';
1878         return;
1879     }
1880     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1881         # gw6 + auto/dhcp
1882         warn "IPv6 gateway specified together with $res->{ip6} address";
1883         return;
1884     }
1885
1886     if (!$res->{ip} && !$res->{ip6}) {
1887         return { ip => 'dhcp', ip6 => 'dhcp' };
1888     }
1889
1890     return $res;
1891 }
1892
1893 sub print_net {
1894     my $net = shift;
1895
1896     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1897 }
1898
1899 sub add_random_macs {
1900     my ($settings) = @_;
1901
1902     foreach my $opt (keys %$settings) {
1903         next if $opt !~ m/^net(\d+)$/;
1904         my $net = parse_net($settings->{$opt});
1905         next if !$net;
1906         $settings->{$opt} = print_net($net);
1907     }
1908 }
1909
1910 sub vm_is_volid_owner {
1911     my ($storecfg, $vmid, $volid) = @_;
1912
1913     if ($volid !~  m|^/|) {
1914         my ($path, $owner);
1915         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1916         if ($owner && ($owner == $vmid)) {
1917             return 1;
1918         }
1919     }
1920
1921     return;
1922 }
1923
1924 sub vmconfig_register_unused_drive {
1925     my ($storecfg, $vmid, $conf, $drive) = @_;
1926
1927     if (drive_is_cloudinit($drive)) {
1928         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1929         warn $@ if $@;
1930         delete $conf->{cloudinit};
1931     } elsif (!drive_is_cdrom($drive)) {
1932         my $volid = $drive->{file};
1933         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1934             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1935         }
1936     }
1937 }
1938
1939 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1940 my $smbios1_fmt = {
1941     uuid => {
1942         type => 'string',
1943         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1944         format_description => 'UUID',
1945         description => "Set SMBIOS1 UUID.",
1946         optional => 1,
1947     },
1948     version => {
1949         type => 'string',
1950         pattern => '[A-Za-z0-9+\/]+={0,2}',
1951         format_description => 'Base64 encoded string',
1952         description => "Set SMBIOS1 version.",
1953         optional => 1,
1954     },
1955     serial => {
1956         type => 'string',
1957         pattern => '[A-Za-z0-9+\/]+={0,2}',
1958         format_description => 'Base64 encoded string',
1959         description => "Set SMBIOS1 serial number.",
1960         optional => 1,
1961     },
1962     manufacturer => {
1963         type => 'string',
1964         pattern => '[A-Za-z0-9+\/]+={0,2}',
1965         format_description => 'Base64 encoded string',
1966         description => "Set SMBIOS1 manufacturer.",
1967         optional => 1,
1968     },
1969     product => {
1970         type => 'string',
1971         pattern => '[A-Za-z0-9+\/]+={0,2}',
1972         format_description => 'Base64 encoded string',
1973         description => "Set SMBIOS1 product ID.",
1974         optional => 1,
1975     },
1976     sku => {
1977         type => 'string',
1978         pattern => '[A-Za-z0-9+\/]+={0,2}',
1979         format_description => 'Base64 encoded string',
1980         description => "Set SMBIOS1 SKU string.",
1981         optional => 1,
1982     },
1983     family => {
1984         type => 'string',
1985         pattern => '[A-Za-z0-9+\/]+={0,2}',
1986         format_description => 'Base64 encoded string',
1987         description => "Set SMBIOS1 family string.",
1988         optional => 1,
1989     },
1990     base64 => {
1991         type => 'boolean',
1992         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1993         optional => 1,
1994     },
1995 };
1996
1997 sub parse_smbios1 {
1998     my ($data) = @_;
1999
2000     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2001     warn $@ if $@;
2002     return $res;
2003 }
2004
2005 sub print_smbios1 {
2006     my ($smbios1) = @_;
2007     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2008 }
2009
2010 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2011
2012 sub parse_watchdog {
2013     my ($value) = @_;
2014
2015     return if !$value;
2016
2017     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2018     warn $@ if $@;
2019     return $res;
2020 }
2021
2022 sub parse_guest_agent {
2023     my ($conf) = @_;
2024
2025     return {} if !defined($conf->{agent});
2026
2027     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2028     warn $@ if $@;
2029
2030     # if the agent is disabled ignore the other potentially set properties
2031     return {} if !$res->{enabled};
2032     return $res;
2033 }
2034
2035 sub get_qga_key {
2036     my ($conf, $key) = @_;
2037     return undef if !defined($conf->{agent});
2038
2039     my $agent = parse_guest_agent($conf);
2040     return $agent->{$key};
2041 }
2042
2043 sub parse_vga {
2044     my ($value) = @_;
2045
2046     return {} if !$value;
2047     my $res = eval { parse_property_string($vga_fmt, $value) };
2048     warn $@ if $@;
2049     return $res;
2050 }
2051
2052 sub parse_rng {
2053     my ($value) = @_;
2054
2055     return if !$value;
2056
2057     my $res = eval { parse_property_string($rng_fmt, $value) };
2058     warn $@ if $@;
2059     return $res;
2060 }
2061
2062 sub parse_meta_info {
2063     my ($value) = @_;
2064
2065     return if !$value;
2066
2067     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2068     warn $@ if $@;
2069     return $res;
2070 }
2071
2072 sub new_meta_info_string {
2073     my () = @_; # for now do not allow to override any value
2074
2075     return PVE::JSONSchema::print_property_string(
2076         {
2077             'creation-qemu' => kvm_user_version(),
2078             ctime => "". int(time()),
2079         },
2080         $meta_info_fmt
2081     );
2082 }
2083
2084 sub qemu_created_version_fixups {
2085     my ($conf, $forcemachine, $kvmver) = @_;
2086
2087     my $meta = parse_meta_info($conf->{meta}) // {};
2088     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2089
2090     # check if we need to apply some handling for VMs that always use the latest machine version but
2091     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2092     # would be required (we do not want to pin machine version for non-windows OS type)
2093     if (
2094         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2095         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2096         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2097         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2098     ) {
2099         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2100         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2101             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2102             # and thus with the predictable interface naming of systemd
2103             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2104         }
2105     }
2106     return;
2107 }
2108
2109 # add JSON properties for create and set function
2110 sub json_config_properties {
2111     my ($prop, $with_disk_alloc) = @_;
2112
2113     my $skip_json_config_opts = {
2114         parent => 1,
2115         snaptime => 1,
2116         vmstate => 1,
2117         runningmachine => 1,
2118         runningcpu => 1,
2119         meta => 1,
2120     };
2121
2122     foreach my $opt (keys %$confdesc) {
2123         next if $skip_json_config_opts->{$opt};
2124
2125         if ($with_disk_alloc && is_valid_drivename($opt)) {
2126             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2127         } else {
2128             $prop->{$opt} = $confdesc->{$opt};
2129         }
2130     }
2131
2132     return $prop;
2133 }
2134
2135 # Properties that we can read from an OVF file
2136 sub json_ovf_properties {
2137     my $prop = {};
2138
2139     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2140         $prop->{$device} = {
2141             type => 'string',
2142             format => 'pve-volume-id-or-absolute-path',
2143             description => "Disk image that gets imported to $device",
2144             optional => 1,
2145         };
2146     }
2147
2148     $prop->{cores} = {
2149         type => 'integer',
2150         description => "The number of CPU cores.",
2151         optional => 1,
2152     };
2153     $prop->{memory} = {
2154         type => 'integer',
2155         description => "Amount of RAM for the VM in MB.",
2156         optional => 1,
2157     };
2158     $prop->{name} = {
2159         type => 'string',
2160         description => "Name of the VM.",
2161         optional => 1,
2162     };
2163
2164     return $prop;
2165 }
2166
2167 # return copy of $confdesc_cloudinit to generate documentation
2168 sub cloudinit_config_properties {
2169
2170     return dclone($confdesc_cloudinit);
2171 }
2172
2173 sub cloudinit_pending_properties {
2174     my $p = {
2175         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2176         name => 1,
2177     };
2178     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2179     return $p;
2180 }
2181
2182 sub check_type {
2183     my ($key, $value) = @_;
2184
2185     die "unknown setting '$key'\n" if !$confdesc->{$key};
2186
2187     my $type = $confdesc->{$key}->{type};
2188
2189     if (!defined($value)) {
2190         die "got undefined value\n";
2191     }
2192
2193     if ($value =~ m/[\n\r]/) {
2194         die "property contains a line feed\n";
2195     }
2196
2197     if ($type eq 'boolean') {
2198         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2199         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2200         die "type check ('boolean') failed - got '$value'\n";
2201     } elsif ($type eq 'integer') {
2202         return int($1) if $value =~ m/^(\d+)$/;
2203         die "type check ('integer') failed - got '$value'\n";
2204     } elsif ($type eq 'number') {
2205         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2206         die "type check ('number') failed - got '$value'\n";
2207     } elsif ($type eq 'string') {
2208         if (my $fmt = $confdesc->{$key}->{format}) {
2209             PVE::JSONSchema::check_format($fmt, $value);
2210             return $value;
2211         }
2212         $value =~ s/^\"(.*)\"$/$1/;
2213         return $value;
2214     } else {
2215         die "internal error"
2216     }
2217 }
2218
2219 sub destroy_vm {
2220     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2221
2222     my $conf = PVE::QemuConfig->load_config($vmid);
2223
2224     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2225         PVE::QemuConfig->check_lock($conf);
2226     }
2227
2228     if ($conf->{template}) {
2229         # check if any base image is still used by a linked clone
2230         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2231                 my ($ds, $drive) = @_;
2232                 return if drive_is_cdrom($drive);
2233
2234                 my $volid = $drive->{file};
2235                 return if !$volid || $volid =~ m|^/|;
2236
2237                 die "base volume '$volid' is still in use by linked cloned\n"
2238                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2239
2240         });
2241     }
2242
2243     my $volids = {};
2244     my $remove_owned_drive = sub {
2245         my ($ds, $drive) = @_;
2246         return if drive_is_cdrom($drive, 1);
2247
2248         my $volid = $drive->{file};
2249         return if !$volid || $volid =~ m|^/|;
2250         return if $volids->{$volid};
2251
2252         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2253         return if !$path || !$owner || ($owner != $vmid);
2254
2255         $volids->{$volid} = 1;
2256         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2257         warn "Could not remove disk '$volid', check manually: $@" if $@;
2258     };
2259
2260     # only remove disks owned by this VM (referenced in the config)
2261     my $include_opts = {
2262         include_unused => 1,
2263         extra_keys => ['vmstate'],
2264     };
2265     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2266
2267     for my $snap (values %{$conf->{snapshots}}) {
2268         next if !defined($snap->{vmstate});
2269         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2270         next if !defined($drive);
2271         $remove_owned_drive->('vmstate', $drive);
2272     }
2273
2274     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2275
2276     if ($purge_unreferenced) { # also remove unreferenced disk
2277         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2278         PVE::Storage::foreach_volid($vmdisks, sub {
2279             my ($volid, $sid, $volname, $d) = @_;
2280             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2281             warn $@ if $@;
2282         });
2283     }
2284
2285     eval { delete_ifaces_ipams_ips($conf, $vmid)};
2286     warn $@ if $@;
2287
2288     if (defined $replacement_conf) {
2289         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2290     } else {
2291         PVE::QemuConfig->destroy_config($vmid);
2292     }
2293 }
2294
2295 sub parse_vm_config {
2296     my ($filename, $raw, $strict) = @_;
2297
2298     return if !defined($raw);
2299
2300     my $res = {
2301         digest => Digest::SHA::sha1_hex($raw),
2302         snapshots => {},
2303         pending => {},
2304         cloudinit => {},
2305     };
2306
2307     my $handle_error = sub {
2308         my ($msg) = @_;
2309
2310         if ($strict) {
2311             die $msg;
2312         } else {
2313             warn $msg;
2314         }
2315     };
2316
2317     $filename =~ m|/qemu-server/(\d+)\.conf$|
2318         || die "got strange filename '$filename'";
2319
2320     my $vmid = $1;
2321
2322     my $conf = $res;
2323     my $descr;
2324     my $finish_description = sub {
2325         if (defined($descr)) {
2326             $descr =~ s/\s+$//;
2327             $conf->{description} = $descr;
2328         }
2329         $descr = undef;
2330     };
2331     my $section = '';
2332
2333     my @lines = split(/\n/, $raw);
2334     foreach my $line (@lines) {
2335         next if $line =~ m/^\s*$/;
2336
2337         if ($line =~ m/^\[PENDING\]\s*$/i) {
2338             $section = 'pending';
2339             $finish_description->();
2340             $conf = $res->{$section} = {};
2341             next;
2342         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2343             $section = 'cloudinit';
2344             $finish_description->();
2345             $conf = $res->{$section} = {};
2346             next;
2347
2348         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2349             $section = $1;
2350             $finish_description->();
2351             $conf = $res->{snapshots}->{$section} = {};
2352             next;
2353         }
2354
2355         if ($line =~ m/^\#(.*)$/) {
2356             $descr = '' if !defined($descr);
2357             $descr .= PVE::Tools::decode_text($1) . "\n";
2358             next;
2359         }
2360
2361         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2362             $descr = '' if !defined($descr);
2363             $descr .= PVE::Tools::decode_text($2);
2364         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2365             $conf->{snapstate} = $1;
2366         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2367             my $key = $1;
2368             my $value = $2;
2369             $conf->{$key} = $value;
2370         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2371             my $value = $1;
2372             if ($section eq 'pending') {
2373                 $conf->{delete} = $value; # we parse this later
2374             } else {
2375                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2376             }
2377         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2378             my $key = $1;
2379             my $value = $2;
2380             if ($section eq 'cloudinit') {
2381                 # ignore validation only used for informative purpose
2382                 $conf->{$key} = $value;
2383                 next;
2384             }
2385             eval { $value = check_type($key, $value); };
2386             if ($@) {
2387                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2388             } else {
2389                 $key = 'ide2' if $key eq 'cdrom';
2390                 my $fmt = $confdesc->{$key}->{format};
2391                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2392                     my $v = parse_drive($key, $value);
2393                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2394                         $v->{file} = $volid;
2395                         $value = print_drive($v);
2396                     } else {
2397                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2398                         next;
2399                     }
2400                 }
2401
2402                 $conf->{$key} = $value;
2403             }
2404         } else {
2405             $handle_error->("vm $vmid - unable to parse config: $line\n");
2406         }
2407     }
2408
2409     $finish_description->();
2410     delete $res->{snapstate}; # just to be sure
2411
2412     return $res;
2413 }
2414
2415 sub write_vm_config {
2416     my ($filename, $conf) = @_;
2417
2418     delete $conf->{snapstate}; # just to be sure
2419
2420     if ($conf->{cdrom}) {
2421         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2422         $conf->{ide2} = $conf->{cdrom};
2423         delete $conf->{cdrom};
2424     }
2425
2426     # we do not use 'smp' any longer
2427     if ($conf->{sockets}) {
2428         delete $conf->{smp};
2429     } elsif ($conf->{smp}) {
2430         $conf->{sockets} = $conf->{smp};
2431         delete $conf->{cores};
2432         delete $conf->{smp};
2433     }
2434
2435     my $used_volids = {};
2436
2437     my $cleanup_config = sub {
2438         my ($cref, $pending, $snapname) = @_;
2439
2440         foreach my $key (keys %$cref) {
2441             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2442                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2443             my $value = $cref->{$key};
2444             if ($key eq 'delete') {
2445                 die "propertry 'delete' is only allowed in [PENDING]\n"
2446                     if !$pending;
2447                 # fixme: check syntax?
2448                 next;
2449             }
2450             eval { $value = check_type($key, $value); };
2451             die "unable to parse value of '$key' - $@" if $@;
2452
2453             $cref->{$key} = $value;
2454
2455             if (!$snapname && is_valid_drivename($key)) {
2456                 my $drive = parse_drive($key, $value);
2457                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2458             }
2459         }
2460     };
2461
2462     &$cleanup_config($conf);
2463
2464     &$cleanup_config($conf->{pending}, 1);
2465
2466     foreach my $snapname (keys %{$conf->{snapshots}}) {
2467         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2468         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2469     }
2470
2471     # remove 'unusedX' settings if we re-add a volume
2472     foreach my $key (keys %$conf) {
2473         my $value = $conf->{$key};
2474         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2475             delete $conf->{$key};
2476         }
2477     }
2478
2479     my $generate_raw_config = sub {
2480         my ($conf, $pending) = @_;
2481
2482         my $raw = '';
2483
2484         # add description as comment to top of file
2485         if (defined(my $descr = $conf->{description})) {
2486             if ($descr) {
2487                 foreach my $cl (split(/\n/, $descr)) {
2488                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2489                 }
2490             } else {
2491                 $raw .= "#\n" if $pending;
2492             }
2493         }
2494
2495         foreach my $key (sort keys %$conf) {
2496             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2497             $raw .= "$key: $conf->{$key}\n";
2498         }
2499         return $raw;
2500     };
2501
2502     my $raw = &$generate_raw_config($conf);
2503
2504     if (scalar(keys %{$conf->{pending}})){
2505         $raw .= "\n[PENDING]\n";
2506         $raw .= &$generate_raw_config($conf->{pending}, 1);
2507     }
2508
2509     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2510         $raw .= "\n[special:cloudinit]\n";
2511         $raw .= &$generate_raw_config($conf->{cloudinit});
2512     }
2513
2514     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2515         $raw .= "\n[$snapname]\n";
2516         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2517     }
2518
2519     return $raw;
2520 }
2521
2522 sub load_defaults {
2523
2524     my $res = {};
2525
2526     # we use static defaults from our JSON schema configuration
2527     foreach my $key (keys %$confdesc) {
2528         if (defined(my $default = $confdesc->{$key}->{default})) {
2529             $res->{$key} = $default;
2530         }
2531     }
2532
2533     return $res;
2534 }
2535
2536 sub config_list {
2537     my $vmlist = PVE::Cluster::get_vmlist();
2538     my $res = {};
2539     return $res if !$vmlist || !$vmlist->{ids};
2540     my $ids = $vmlist->{ids};
2541     my $nodename = nodename();
2542
2543     foreach my $vmid (keys %$ids) {
2544         my $d = $ids->{$vmid};
2545         next if !$d->{node} || $d->{node} ne $nodename;
2546         next if !$d->{type} || $d->{type} ne 'qemu';
2547         $res->{$vmid}->{exists} = 1;
2548     }
2549     return $res;
2550 }
2551
2552 # test if VM uses local resources (to prevent migration)
2553 sub check_local_resources {
2554     my ($conf, $noerr) = @_;
2555
2556     my @loc_res = ();
2557     my $mapped_res = [];
2558
2559     my $nodelist = PVE::Cluster::get_nodelist();
2560     my $pci_map = PVE::Mapping::PCI::config();
2561     my $usb_map = PVE::Mapping::USB::config();
2562
2563     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2564
2565     my $add_missing_mapping = sub {
2566         my ($type, $key, $id) = @_;
2567         for my $node (@$nodelist) {
2568             my $entry;
2569             if ($type eq 'pci') {
2570                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2571             } elsif ($type eq 'usb') {
2572                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2573             }
2574             if (!scalar($entry->@*)) {
2575                 push @{$missing_mappings_by_node->{$node}}, $key;
2576             }
2577         }
2578     };
2579
2580     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2581     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2582
2583     push @loc_res, "ivshmem" if $conf->{ivshmem};
2584
2585     foreach my $k (keys %$conf) {
2586         if ($k =~ m/^usb/) {
2587             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2588             next if $entry->{host} =~ m/^spice$/i;
2589             if ($entry->{mapping}) {
2590                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2591                 push @$mapped_res, $k;
2592             }
2593         }
2594         if ($k =~ m/^hostpci/) {
2595             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2596             if ($entry->{mapping}) {
2597                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2598                 push @$mapped_res, $k;
2599             }
2600         }
2601         # sockets are safe: they will recreated be on the target side post-migrate
2602         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2603         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2604     }
2605
2606     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2607
2608     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2609 }
2610
2611 # check if used storages are available on all nodes (use by migrate)
2612 sub check_storage_availability {
2613     my ($storecfg, $conf, $node) = @_;
2614
2615     PVE::QemuConfig->foreach_volume($conf, sub {
2616         my ($ds, $drive) = @_;
2617
2618         my $volid = $drive->{file};
2619         return if !$volid;
2620
2621         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2622         return if !$sid;
2623
2624         # check if storage is available on both nodes
2625         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2626         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2627
2628         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2629
2630         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2631             if !$scfg->{content}->{$vtype};
2632    });
2633 }
2634
2635 # list nodes where all VM images are available (used by has_feature API)
2636 sub shared_nodes {
2637     my ($conf, $storecfg) = @_;
2638
2639     my $nodelist = PVE::Cluster::get_nodelist();
2640     my $nodehash = { map { $_ => 1 } @$nodelist };
2641     my $nodename = nodename();
2642
2643     PVE::QemuConfig->foreach_volume($conf, sub {
2644         my ($ds, $drive) = @_;
2645
2646         my $volid = $drive->{file};
2647         return if !$volid;
2648
2649         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2650         if ($storeid) {
2651             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2652             if ($scfg->{disable}) {
2653                 $nodehash = {};
2654             } elsif (my $avail = $scfg->{nodes}) {
2655                 foreach my $node (keys %$nodehash) {
2656                     delete $nodehash->{$node} if !$avail->{$node};
2657                 }
2658             } elsif (!$scfg->{shared}) {
2659                 foreach my $node (keys %$nodehash) {
2660                     delete $nodehash->{$node} if $node ne $nodename
2661                 }
2662             }
2663         }
2664     });
2665
2666     return $nodehash
2667 }
2668
2669 sub check_local_storage_availability {
2670     my ($conf, $storecfg) = @_;
2671
2672     my $nodelist = PVE::Cluster::get_nodelist();
2673     my $nodehash = { map { $_ => {} } @$nodelist };
2674
2675     PVE::QemuConfig->foreach_volume($conf, sub {
2676         my ($ds, $drive) = @_;
2677
2678         my $volid = $drive->{file};
2679         return if !$volid;
2680
2681         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2682         if ($storeid) {
2683             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2684
2685             if ($scfg->{disable}) {
2686                 foreach my $node (keys %$nodehash) {
2687                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2688                 }
2689             } elsif (my $avail = $scfg->{nodes}) {
2690                 foreach my $node (keys %$nodehash) {
2691                     if (!$avail->{$node}) {
2692                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2693                     }
2694                 }
2695             }
2696         }
2697     });
2698
2699     foreach my $node (values %$nodehash) {
2700         if (my $unavail = $node->{unavailable_storages}) {
2701             $node->{unavailable_storages} = [ sort keys %$unavail ];
2702         }
2703     }
2704
2705     return $nodehash
2706 }
2707
2708 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2709 sub check_running {
2710     my ($vmid, $nocheck, $node) = @_;
2711
2712     # $nocheck is set when called during a migration, in which case the config
2713     # file might still or already reside on the *other* node
2714     # - because rename has already happened, and current node is source
2715     # - because rename hasn't happened yet, and current node is target
2716     # - because rename has happened, current node is target, but hasn't yet
2717     # processed it yet
2718     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2719     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2720 }
2721
2722 sub vzlist {
2723
2724     my $vzlist = config_list();
2725
2726     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2727
2728     while (defined(my $de = $fd->read)) {
2729         next if $de !~ m/^(\d+)\.pid$/;
2730         my $vmid = $1;
2731         next if !defined($vzlist->{$vmid});
2732         if (my $pid = check_running($vmid)) {
2733             $vzlist->{$vmid}->{pid} = $pid;
2734         }
2735     }
2736
2737     return $vzlist;
2738 }
2739
2740 our $vmstatus_return_properties = {
2741     vmid => get_standard_option('pve-vmid'),
2742     status => {
2743         description => "QEMU process status.",
2744         type => 'string',
2745         enum => ['stopped', 'running'],
2746     },
2747     maxmem => {
2748         description => "Maximum memory in bytes.",
2749         type => 'integer',
2750         optional => 1,
2751         renderer => 'bytes',
2752     },
2753     maxdisk => {
2754         description => "Root disk size in bytes.",
2755         type => 'integer',
2756         optional => 1,
2757         renderer => 'bytes',
2758     },
2759     name => {
2760         description => "VM name.",
2761         type => 'string',
2762         optional => 1,
2763     },
2764     qmpstatus => {
2765         description => "VM run state from the 'query-status' QMP monitor command.",
2766         type => 'string',
2767         optional => 1,
2768     },
2769     pid => {
2770         description => "PID of running qemu process.",
2771         type => 'integer',
2772         optional => 1,
2773     },
2774     uptime => {
2775         description => "Uptime.",
2776         type => 'integer',
2777         optional => 1,
2778         renderer => 'duration',
2779     },
2780     cpus => {
2781         description => "Maximum usable CPUs.",
2782         type => 'number',
2783         optional => 1,
2784     },
2785     lock => {
2786         description => "The current config lock, if any.",
2787         type => 'string',
2788         optional => 1,
2789     },
2790     tags => {
2791         description  => "The current configured tags, if any",
2792         type => 'string',
2793         optional => 1,
2794     },
2795     'running-machine' => {
2796         description => "The currently running machine type (if running).",
2797         type => 'string',
2798         optional => 1,
2799     },
2800     'running-qemu' => {
2801         description => "The currently running QEMU version (if running).",
2802         type => 'string',
2803         optional => 1,
2804     },
2805 };
2806
2807 my $last_proc_pid_stat;
2808
2809 # get VM status information
2810 # This must be fast and should not block ($full == false)
2811 # We only query KVM using QMP if $full == true (this can be slow)
2812 sub vmstatus {
2813     my ($opt_vmid, $full) = @_;
2814
2815     my $res = {};
2816
2817     my $storecfg = PVE::Storage::config();
2818
2819     my $list = vzlist();
2820     my $defaults = load_defaults();
2821
2822     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2823
2824     my $cpucount = $cpuinfo->{cpus} || 1;
2825
2826     foreach my $vmid (keys %$list) {
2827         next if $opt_vmid && ($vmid ne $opt_vmid);
2828
2829         my $conf = PVE::QemuConfig->load_config($vmid);
2830
2831         my $d = { vmid => int($vmid) };
2832         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2833
2834         # fixme: better status?
2835         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2836
2837         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2838         if (defined($size)) {
2839             $d->{disk} = 0; # no info available
2840             $d->{maxdisk} = $size;
2841         } else {
2842             $d->{disk} = 0;
2843             $d->{maxdisk} = 0;
2844         }
2845
2846         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2847             * ($conf->{cores} || $defaults->{cores});
2848         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2849         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2850
2851         $d->{name} = $conf->{name} || "VM $vmid";
2852         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2853
2854         if ($conf->{balloon}) {
2855             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2856             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2857                 : $defaults->{shares};
2858         }
2859
2860         $d->{uptime} = 0;
2861         $d->{cpu} = 0;
2862         $d->{mem} = 0;
2863
2864         $d->{netout} = 0;
2865         $d->{netin} = 0;
2866
2867         $d->{diskread} = 0;
2868         $d->{diskwrite} = 0;
2869
2870         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2871
2872         $d->{serial} = 1 if conf_has_serial($conf);
2873         $d->{lock} = $conf->{lock} if $conf->{lock};
2874         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2875
2876         $res->{$vmid} = $d;
2877     }
2878
2879     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2880     foreach my $dev (keys %$netdev) {
2881         next if $dev !~ m/^tap([1-9]\d*)i/;
2882         my $vmid = $1;
2883         my $d = $res->{$vmid};
2884         next if !$d;
2885
2886         $d->{netout} += $netdev->{$dev}->{receive};
2887         $d->{netin} += $netdev->{$dev}->{transmit};
2888
2889         if ($full) {
2890             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2891             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2892         }
2893
2894     }
2895
2896     my $ctime = gettimeofday;
2897
2898     foreach my $vmid (keys %$list) {
2899
2900         my $d = $res->{$vmid};
2901         my $pid = $d->{pid};
2902         next if !$pid;
2903
2904         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2905         next if !$pstat; # not running
2906
2907         my $used = $pstat->{utime} + $pstat->{stime};
2908
2909         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2910
2911         if ($pstat->{vsize}) {
2912             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2913         }
2914
2915         my $old = $last_proc_pid_stat->{$pid};
2916         if (!$old) {
2917             $last_proc_pid_stat->{$pid} = {
2918                 time => $ctime,
2919                 used => $used,
2920                 cpu => 0,
2921             };
2922             next;
2923         }
2924
2925         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2926
2927         if ($dtime > 1000) {
2928             my $dutime = $used -  $old->{used};
2929
2930             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2931             $last_proc_pid_stat->{$pid} = {
2932                 time => $ctime,
2933                 used => $used,
2934                 cpu => $d->{cpu},
2935             };
2936         } else {
2937             $d->{cpu} = $old->{cpu};
2938         }
2939     }
2940
2941     return $res if !$full;
2942
2943     my $qmpclient = PVE::QMPClient->new();
2944
2945     my $ballooncb = sub {
2946         my ($vmid, $resp) = @_;
2947
2948         my $info = $resp->{'return'};
2949         return if !$info->{max_mem};
2950
2951         my $d = $res->{$vmid};
2952
2953         # use memory assigned to VM
2954         $d->{maxmem} = $info->{max_mem};
2955         $d->{balloon} = $info->{actual};
2956
2957         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2958             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2959             $d->{freemem} = $info->{free_mem};
2960         }
2961
2962         $d->{ballooninfo} = $info;
2963     };
2964
2965     my $blockstatscb = sub {
2966         my ($vmid, $resp) = @_;
2967         my $data = $resp->{'return'} || [];
2968         my $totalrdbytes = 0;
2969         my $totalwrbytes = 0;
2970
2971         for my $blockstat (@$data) {
2972             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2973             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2974
2975             $blockstat->{device} =~ s/drive-//;
2976             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2977         }
2978         $res->{$vmid}->{diskread} = $totalrdbytes;
2979         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2980     };
2981
2982     my $machinecb = sub {
2983         my ($vmid, $resp) = @_;
2984         my $data = $resp->{'return'} || [];
2985
2986         $res->{$vmid}->{'running-machine'} =
2987             PVE::QemuServer::Machine::current_from_query_machines($data);
2988     };
2989
2990     my $versioncb = sub {
2991         my ($vmid, $resp) = @_;
2992         my $data = $resp->{'return'} // {};
2993         my $version = 'unknown';
2994
2995         if (my $v = $data->{qemu}) {
2996             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2997         }
2998
2999         $res->{$vmid}->{'running-qemu'} = $version;
3000     };
3001
3002     my $statuscb = sub {
3003         my ($vmid, $resp) = @_;
3004
3005         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3006         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3007         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3008         # this fails if ballon driver is not loaded, so this must be
3009         # the last commnand (following command are aborted if this fails).
3010         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3011
3012         my $status = 'unknown';
3013         if (!defined($status = $resp->{'return'}->{status})) {
3014             warn "unable to get VM status\n";
3015             return;
3016         }
3017
3018         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3019     };
3020
3021     foreach my $vmid (keys %$list) {
3022         next if $opt_vmid && ($vmid ne $opt_vmid);
3023         next if !$res->{$vmid}->{pid}; # not running
3024         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3025     }
3026
3027     $qmpclient->queue_execute(undef, 2);
3028
3029     foreach my $vmid (keys %$list) {
3030         next if $opt_vmid && ($vmid ne $opt_vmid);
3031         next if !$res->{$vmid}->{pid}; #not running
3032
3033         # we can't use the $qmpclient since it might have already aborted on
3034         # 'query-balloon', but this might also fail for older versions...
3035         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3036         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3037     }
3038
3039     foreach my $vmid (keys %$list) {
3040         next if $opt_vmid && ($vmid ne $opt_vmid);
3041         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3042     }
3043
3044     return $res;
3045 }
3046
3047 sub conf_has_serial {
3048     my ($conf) = @_;
3049
3050     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3051         if ($conf->{"serial$i"}) {
3052             return 1;
3053         }
3054     }
3055
3056     return 0;
3057 }
3058
3059 sub conf_has_audio {
3060     my ($conf, $id) = @_;
3061
3062     $id //= 0;
3063     my $audio = $conf->{"audio$id"};
3064     return if !defined($audio);
3065
3066     my $audioproperties = parse_property_string($audio_fmt, $audio);
3067     my $audiodriver = $audioproperties->{driver} // 'spice';
3068
3069     return {
3070         dev => $audioproperties->{device},
3071         dev_id => "audiodev$id",
3072         backend => $audiodriver,
3073         backend_id => "$audiodriver-backend${id}",
3074     };
3075 }
3076
3077 sub audio_devs {
3078     my ($audio, $audiopciaddr, $machine_version) = @_;
3079
3080     my $devs = [];
3081
3082     my $id = $audio->{dev_id};
3083     my $audiodev = "";
3084     if (min_version($machine_version, 4, 2)) {
3085         $audiodev = ",audiodev=$audio->{backend_id}";
3086     }
3087
3088     if ($audio->{dev} eq 'AC97') {
3089         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3090     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3091         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3092         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3093         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3094     } else {
3095         die "unkown audio device '$audio->{dev}', implement me!";
3096     }
3097
3098     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3099
3100     return $devs;
3101 }
3102
3103 sub get_tpm_paths {
3104     my ($vmid) = @_;
3105     return {
3106         socket => "/var/run/qemu-server/$vmid.swtpm",
3107         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3108     };
3109 }
3110
3111 sub add_tpm_device {
3112     my ($vmid, $devices, $conf) = @_;
3113
3114     return if !$conf->{tpmstate0};
3115
3116     my $paths = get_tpm_paths($vmid);
3117
3118     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3119     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3120     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3121 }
3122
3123 sub start_swtpm {
3124     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3125
3126     return if !$tpmdrive;
3127
3128     my $state;
3129     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3130     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3131     if ($storeid) {
3132         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3133     } else {
3134         $state = $tpm->{file};
3135     }
3136
3137     my $paths = get_tpm_paths($vmid);
3138
3139     # during migration, we will get state from remote
3140     #
3141     if (!$migration) {
3142         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3143         my $setup_cmd = [
3144             "swtpm_setup",
3145             "--tpmstate",
3146             "file://$state",
3147             "--createek",
3148             "--create-ek-cert",
3149             "--create-platform-cert",
3150             "--lock-nvram",
3151             "--config",
3152             "/etc/swtpm_setup.conf", # do not use XDG configs
3153             "--runas",
3154             "0", # force creation as root, error if not possible
3155             "--not-overwrite", # ignore existing state, do not modify
3156         ];
3157
3158         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3159         # TPM 2.0 supports ECC crypto, use if possible
3160         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3161
3162         run_command($setup_cmd, outfunc => sub {
3163             print "swtpm_setup: $1\n";
3164         });
3165     }
3166
3167     # Used to distinguish different invocations in the log.
3168     my $log_prefix = "[id=" . int(time()) . "] ";
3169
3170     my $emulator_cmd = [
3171         "swtpm",
3172         "socket",
3173         "--tpmstate",
3174         "backend-uri=file://$state,mode=0600",
3175         "--ctrl",
3176         "type=unixio,path=$paths->{socket},mode=0600",
3177         "--pid",
3178         "file=$paths->{pid}",
3179         "--terminate", # terminate on QEMU disconnect
3180         "--daemon",
3181         "--log",
3182         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3183     ];
3184     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3185     run_command($emulator_cmd, outfunc => sub { print $1; });
3186
3187     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3188     while (! -e $paths->{pid}) {
3189         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3190         usleep(50_000);
3191     }
3192
3193     # return untainted PID of swtpm daemon so it can be killed on error
3194     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3195     return $1;
3196 }
3197
3198 sub vga_conf_has_spice {
3199     my ($vga) = @_;
3200
3201     my $vgaconf = parse_vga($vga);
3202     my $vgatype = $vgaconf->{type};
3203     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3204
3205     return $1 || 1;
3206 }
3207
3208 sub is_native($) {
3209     my ($arch) = @_;
3210     return get_host_arch() eq $arch;
3211 }
3212
3213 sub get_vm_arch {
3214     my ($conf) = @_;
3215     return $conf->{arch} // get_host_arch();
3216 }
3217
3218 my $default_machines = {
3219     x86_64 => 'pc',
3220     aarch64 => 'virt',
3221 };
3222
3223 sub get_installed_machine_version {
3224     my ($kvmversion) = @_;
3225     $kvmversion = kvm_user_version() if !defined($kvmversion);
3226     $kvmversion =~ m/^(\d+\.\d+)/;
3227     return $1;
3228 }
3229
3230 sub windows_get_pinned_machine_version {
3231     my ($machine, $base_version, $kvmversion) = @_;
3232
3233     my $pin_version = $base_version;
3234     if (!defined($base_version) ||
3235         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3236     ) {
3237         $pin_version = get_installed_machine_version($kvmversion);
3238     }
3239     if (!$machine || $machine eq 'pc') {
3240         $machine = "pc-i440fx-$pin_version";
3241     } elsif ($machine eq 'q35') {
3242         $machine = "pc-q35-$pin_version";
3243     } elsif ($machine eq 'virt') {
3244         $machine = "virt-$pin_version";
3245     } else {
3246         warn "unknown machine type '$machine', not touching that!\n";
3247     }
3248
3249     return $machine;
3250 }
3251
3252 sub get_vm_machine {
3253     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3254
3255     my $machine = $forcemachine || $conf->{machine};
3256
3257     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3258         $kvmversion //= kvm_user_version();
3259         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3260         # layout which confuses windows quite a bit and may result in various regressions..
3261         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3262         if (windows_version($conf->{ostype})) {
3263             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3264         }
3265         $arch //= 'x86_64';
3266         $machine ||= $default_machines->{$arch};
3267         if ($add_pve_version) {
3268             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3269             $machine .= "+pve$pvever";
3270         }
3271     }
3272
3273     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3274         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3275         $machine = $1 if $is_pxe;
3276
3277         # for version-pinned machines that do not include a pve-version (e.g.
3278         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3279         $machine .= '+pve0';
3280
3281         $machine .= '.pxe' if $is_pxe;
3282     }
3283
3284     return $machine;
3285 }
3286
3287 sub get_ovmf_files($$$) {
3288     my ($arch, $efidisk, $smm) = @_;
3289
3290     my $types = $OVMF->{$arch}
3291         or die "no OVMF images known for architecture '$arch'\n";
3292
3293     my $type = 'default';
3294     if ($arch eq 'x86_64') {
3295         if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3296             $type = $smm ? "4m" : "4m-no-smm";
3297             $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3298         } else {
3299             # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3300         }
3301     }
3302
3303     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3304     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3305     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3306
3307     return ($ovmf_code, $ovmf_vars);
3308 }
3309
3310 my $Arch2Qemu = {
3311     aarch64 => '/usr/bin/qemu-system-aarch64',
3312     x86_64 => '/usr/bin/qemu-system-x86_64',
3313 };
3314 sub get_command_for_arch($) {
3315     my ($arch) = @_;
3316     return '/usr/bin/kvm' if is_native($arch);
3317
3318     my $cmd = $Arch2Qemu->{$arch}
3319         or die "don't know how to emulate architecture '$arch'\n";
3320     return $cmd;
3321 }
3322
3323 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3324 # to use in a QEMU command line (-cpu element), first array_intersect the result
3325 # of query_supported_ with query_understood_. This is necessary because:
3326 #
3327 # a) query_understood_ returns flags the host cannot use and
3328 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3329 #    flags, but CPU settings - with most of them being flags. Those settings
3330 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3331 #
3332 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3333 # expensive. If you need the value returned from this, you can get it much
3334 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3335 # $accel being 'kvm' or 'tcg'.
3336 #
3337 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3338 # changes, automatically populating pmxcfs.
3339 #
3340 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3341 # since kvm and tcg machines support different flags
3342 #
3343 sub query_supported_cpu_flags {
3344     my ($arch) = @_;
3345
3346     $arch //= get_host_arch();
3347     my $default_machine = $default_machines->{$arch};
3348
3349     my $flags = {};
3350
3351     # FIXME: Once this is merged, the code below should work for ARM as well:
3352     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3353     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3354         $arch eq "aarch64";
3355
3356     my $kvm_supported = defined(kvm_version());
3357     my $qemu_cmd = get_command_for_arch($arch);
3358     my $fakevmid = -1;
3359     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3360
3361     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3362     my $query_supported_run_qemu = sub {
3363         my ($kvm) = @_;
3364
3365         my $flags = {};
3366         my $cmd = [
3367             $qemu_cmd,
3368             '-machine', $default_machine,
3369             '-display', 'none',
3370             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3371             '-mon', 'chardev=qmp,mode=control',
3372             '-pidfile', $pidfile,
3373             '-S', '-daemonize'
3374         ];
3375
3376         if (!$kvm) {
3377             push @$cmd, '-accel', 'tcg';
3378         }
3379
3380         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3381         die "QEMU flag querying VM exited with code " . $rc if $rc;
3382
3383         eval {
3384             my $cmd_result = mon_cmd(
3385                 $fakevmid,
3386                 'query-cpu-model-expansion',
3387                 type => 'full',
3388                 model => { name => 'host' }
3389             );
3390
3391             my $props = $cmd_result->{model}->{props};
3392             foreach my $prop (keys %$props) {
3393                 next if $props->{$prop} ne '1';
3394                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3395                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3396                 # We only keep those with underscores, to match /proc/cpuinfo
3397                 $prop =~ s/\.|-/_/g;
3398                 $flags->{$prop} = 1;
3399             }
3400         };
3401         my $err = $@;
3402
3403         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3404         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3405
3406         die $err if $err;
3407
3408         return [ sort keys %$flags ];
3409     };
3410
3411     # We need to query QEMU twice, since KVM and TCG have different supported flags
3412     PVE::QemuConfig->lock_config($fakevmid, sub {
3413         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3414         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3415
3416         if ($kvm_supported) {
3417             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3418             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3419         }
3420     });
3421
3422     return $flags;
3423 }
3424
3425 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3426 my $understood_cpu_flag_dir = "/usr/share/kvm";
3427 sub query_understood_cpu_flags {
3428     my $arch = get_host_arch();
3429     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3430
3431     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3432         if ! -e $filepath;
3433
3434     my $raw = file_get_contents($filepath);
3435     $raw =~ s/^\s+|\s+$//g;
3436     my @flags = split(/\s+/, $raw);
3437
3438     return \@flags;
3439 }
3440
3441 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3442 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3443 my sub should_disable_smm {
3444     my ($conf, $vga, $machine) = @_;
3445
3446     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3447
3448     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3449         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3450 }
3451
3452 my sub print_ovmf_drive_commandlines {
3453     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3454
3455     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3456
3457     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3458
3459     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3460     if ($d) {
3461         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3462         my ($path, $format) = $d->@{'file', 'format'};
3463         if ($storeid) {
3464             $path = PVE::Storage::path($storecfg, $d->{file});
3465             if (!defined($format)) {
3466                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3467                 $format = qemu_img_format($scfg, $volname);
3468             }
3469         } elsif (!defined($format)) {
3470             die "efidisk format must be specified\n";
3471         }
3472         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3473         if ($path =~ m/^rbd:/) {
3474             $var_drive_str .= ',cache=writeback';
3475             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3476         }
3477         $var_drive_str .= ",format=$format,file=$path";
3478
3479         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3480         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3481     } else {
3482         log_warn("no efidisk configured! Using temporary efivars disk.");
3483         my $path = "/tmp/$vmid-ovmf.fd";
3484         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3485         $var_drive_str .= ",format=raw,file=$path";
3486         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3487     }
3488
3489     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3490 }
3491
3492 sub config_to_command {
3493     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3494         $pbs_backing) = @_;
3495
3496     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3497     my $devices = [];
3498     my $bridges = {};
3499     my $ostype = $conf->{ostype};
3500     my $winversion = windows_version($ostype);
3501     my $kvm = $conf->{kvm};
3502     my $nodename = nodename();
3503
3504     my $arch = get_vm_arch($conf);
3505     my $kvm_binary = get_command_for_arch($arch);
3506     my $kvmver = kvm_user_version($kvm_binary);
3507
3508     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3509         $kvmver //= "undefined";
3510         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3511     }
3512
3513     my $add_pve_version = min_version($kvmver, 4, 1);
3514
3515     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3516     my $machine_version = extract_version($machine_type, $kvmver);
3517     $kvm //= 1 if is_native($arch);
3518
3519     $machine_version =~ m/(\d+)\.(\d+)/;
3520     my ($machine_major, $machine_minor) = ($1, $2);
3521
3522     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3523         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3524     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3525         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3526             ." please upgrade node '$nodename'\n"
3527     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3528         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3529         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3530             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3531             ." node '$nodename'\n";
3532     }
3533
3534     # if a specific +pve version is required for a feature, use $version_guard
3535     # instead of min_version to allow machines to be run with the minimum
3536     # required version
3537     my $required_pve_version = 0;
3538     my $version_guard = sub {
3539         my ($major, $minor, $pve) = @_;
3540         return 0 if !min_version($machine_version, $major, $minor, $pve);
3541         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3542         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3543         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3544         return 1;
3545     };
3546
3547     if ($kvm && !defined kvm_version()) {
3548         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3549             ." or enable in BIOS.\n";
3550     }
3551
3552     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3553     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3554     my $use_old_bios_files = undef;
3555     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3556
3557     my $cmd = [];
3558     if ($conf->{affinity}) {
3559         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3560     }
3561
3562     push @$cmd, $kvm_binary;
3563
3564     push @$cmd, '-id', $vmid;
3565
3566     my $vmname = $conf->{name} || "vm$vmid";
3567
3568     push @$cmd, '-name', "$vmname,debug-threads=on";
3569
3570     push @$cmd, '-no-shutdown';
3571
3572     my $use_virtio = 0;
3573
3574     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3575     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3576     push @$cmd, '-mon', "chardev=qmp,mode=control";
3577
3578     if (min_version($machine_version, 2, 12)) {
3579         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3580         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3581     }
3582
3583     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3584
3585     push @$cmd, '-daemonize';
3586
3587     if ($conf->{smbios1}) {
3588         my $smbios_conf = parse_smbios1($conf->{smbios1});
3589         if ($smbios_conf->{base64}) {
3590             # Do not pass base64 flag to qemu
3591             delete $smbios_conf->{base64};
3592             my $smbios_string = "";
3593             foreach my $key (keys %$smbios_conf) {
3594                 my $value;
3595                 if ($key eq "uuid") {
3596                     $value = $smbios_conf->{uuid}
3597                 } else {
3598                     $value = decode_base64($smbios_conf->{$key});
3599                 }
3600                 # qemu accepts any binary data, only commas need escaping by double comma
3601                 $value =~ s/,/,,/g;
3602                 $smbios_string .= "," . $key . "=" . $value if $value;
3603             }
3604             push @$cmd, '-smbios', "type=1" . $smbios_string;
3605         } else {
3606             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3607         }
3608     }
3609
3610     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3611         my ($code_drive_str, $var_drive_str) =
3612             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3613         push $cmd->@*, '-drive', $code_drive_str;
3614         push $cmd->@*, '-drive', $var_drive_str;
3615     }
3616
3617     if ($q35) { # tell QEMU to load q35 config early
3618         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3619         if (min_version($machine_version, 4, 0)) {
3620             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3621         } else {
3622             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3623         }
3624     }
3625
3626     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3627         push @$cmd, $fixups->@*;
3628     }
3629
3630     if ($conf->{vmgenid}) {
3631         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3632     }
3633
3634     # add usb controllers
3635     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3636         $conf, $bridges, $arch, $machine_type, $machine_version);
3637     push @$devices, @usbcontrollers if @usbcontrollers;
3638     my $vga = parse_vga($conf->{vga});
3639
3640     my $qxlnum = vga_conf_has_spice($conf->{vga});
3641     $vga->{type} = 'qxl' if $qxlnum;
3642
3643     if (!$vga->{type}) {
3644         if ($arch eq 'aarch64') {
3645             $vga->{type} = 'virtio';
3646         } elsif (min_version($machine_version, 2, 9)) {
3647             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3648         } else {
3649             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3650         }
3651     }
3652
3653     # enable absolute mouse coordinates (needed by vnc)
3654     my $tablet = $conf->{tablet};
3655     if (!defined($tablet)) {
3656         $tablet = $defaults->{tablet};
3657         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3658         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3659     }
3660
3661     if ($tablet) {
3662         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3663         my $kbd = print_keyboarddevice_full($conf, $arch);
3664         push @$devices, '-device', $kbd if defined($kbd);
3665     }
3666
3667     my $bootorder = device_bootorder($conf);
3668
3669     # host pci device passthrough
3670     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3671         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3672
3673     # usb devices
3674     my $usb_dev_features = {};
3675     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3676
3677     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3678         $conf, $usb_dev_features, $bootorder, $machine_version);
3679     push @$devices, @usbdevices if @usbdevices;
3680
3681     # serial devices
3682     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3683         my $path = $conf->{"serial$i"} or next;
3684         if ($path eq 'socket') {
3685             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3686             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3687             # On aarch64, serial0 is the UART device. QEMU only allows
3688             # connecting UART devices via the '-serial' command line, as
3689             # the device has a fixed slot on the hardware...
3690             if ($arch eq 'aarch64' && $i == 0) {
3691                 push @$devices, '-serial', "chardev:serial$i";
3692             } else {
3693                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3694             }
3695         } else {
3696             die "no such serial device\n" if ! -c $path;
3697             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3698             push @$devices, '-device', "isa-serial,chardev=serial$i";
3699         }
3700     }
3701
3702     # parallel devices
3703     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3704         if (my $path = $conf->{"parallel$i"}) {
3705             die "no such parallel device\n" if ! -c $path;
3706             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3707             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3708             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3709         }
3710     }
3711
3712     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3713         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3714         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3715         push @$devices, @$audio_devs;
3716     }
3717
3718     # Add a TPM only if the VM is not a template,
3719     # to support backing up template VMs even if the TPM disk is write-protected.
3720     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3721
3722     my $sockets = 1;
3723     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3724     $sockets = $conf->{sockets} if  $conf->{sockets};
3725
3726     my $cores = $conf->{cores} || 1;
3727
3728     my $maxcpus = $sockets * $cores;
3729
3730     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3731
3732     my $allowed_vcpus = $cpuinfo->{cpus};
3733
3734     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3735
3736     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3737         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3738         for (my $i = 2; $i <= $vcpus; $i++)  {
3739             my $cpustr = print_cpu_device($conf,$i);
3740             push @$cmd, '-device', $cpustr;
3741         }
3742
3743     } else {
3744
3745         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3746     }
3747     push @$cmd, '-nodefaults';
3748
3749     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3750
3751     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3752
3753     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3754
3755     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3756         push @$devices, '-device', print_vga_device(
3757             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3758
3759         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3760
3761         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3762         push @$cmd,  '-vnc', "unix:$socket,password=on";
3763     } else {
3764         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3765         push @$cmd, '-nographic';
3766     }
3767
3768     # time drift fix
3769     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3770     my $useLocaltime = $conf->{localtime};
3771
3772     if ($winversion >= 5) { # windows
3773         $useLocaltime = 1 if !defined($conf->{localtime});
3774
3775         # use time drift fix when acpi is enabled
3776         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3777             $tdf = 1 if !defined($conf->{tdf});
3778         }
3779     }
3780
3781     if ($winversion >= 6) {
3782         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3783         push @$machineFlags, 'hpet=off';
3784     }
3785
3786     push @$rtcFlags, 'driftfix=slew' if $tdf;
3787
3788     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3789         push @$rtcFlags, "base=$conf->{startdate}";
3790     } elsif ($useLocaltime) {
3791         push @$rtcFlags, 'base=localtime';
3792     }
3793
3794     if ($forcecpu) {
3795         push @$cmd, '-cpu', $forcecpu;
3796     } else {
3797         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3798     }
3799
3800     PVE::QemuServer::Memory::config(
3801         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3802
3803     push @$cmd, '-S' if $conf->{freeze};
3804
3805     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3806
3807     my $guest_agent = parse_guest_agent($conf);
3808
3809     if ($guest_agent->{enabled}) {
3810         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3811         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3812
3813         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3814             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3815             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3816             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3817         } elsif ($guest_agent->{type} eq 'isa') {
3818             push @$devices, '-device', "isa-serial,chardev=qga0";
3819         }
3820     }
3821
3822     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3823     if ($rng && $version_guard->(4, 1, 2)) {
3824         check_rng_source($rng->{source});
3825
3826         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3827         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3828         my $limiter_str = "";
3829         if ($max_bytes) {
3830             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3831         }
3832
3833         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3834         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3835         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3836     }
3837
3838     my $spice_port;
3839
3840     assert_clipboard_config($vga);
3841     my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3842
3843     if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3844         if ($qxlnum > 1) {
3845             if ($winversion){
3846                 for (my $i = 1; $i < $qxlnum; $i++){
3847                     push @$devices, '-device', print_vga_device(
3848                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3849                 }
3850             } else {
3851                 # assume other OS works like Linux
3852                 my ($ram, $vram) = ("134217728", "67108864");
3853                 if ($vga->{memory}) {
3854                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3855                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3856                 }
3857                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3858                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3859             }
3860         }
3861
3862         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3863
3864         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3865         if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3866             push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3867         } else {
3868             push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3869         }
3870         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3871
3872         if ($is_spice) {
3873             my $pfamily = PVE::Tools::get_host_address_family($nodename);
3874             my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3875             die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3876
3877             my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3878             $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3879
3880             my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3881             my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3882             if ($spice_enhancement->{foldersharing}) {
3883                 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3884                 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3885             }
3886
3887             my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3888             $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3889                 if $spice_enhancement->{videostreaming};
3890             push @$devices, '-spice', "$spice_opts";
3891         }
3892     }
3893
3894     # enable balloon by default, unless explicitly disabled
3895     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3896         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3897         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3898         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3899         push @$devices, '-device', $ballooncmd;
3900     }
3901
3902     if ($conf->{watchdog}) {
3903         my $wdopts = parse_watchdog($conf->{watchdog});
3904         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3905         my $watchdog = $wdopts->{model} || 'i6300esb';
3906         push @$devices, '-device', "$watchdog$pciaddr";
3907         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3908     }
3909
3910     my $vollist = [];
3911     my $scsicontroller = {};
3912     my $ahcicontroller = {};
3913     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3914
3915     # Add iscsi initiator name if available
3916     if (my $initiator = get_initiator_name()) {
3917         push @$devices, '-iscsi', "initiator-name=$initiator";
3918     }
3919
3920     PVE::QemuConfig->foreach_volume($conf, sub {
3921         my ($ds, $drive) = @_;
3922
3923         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3924             check_volume_storage_type($storecfg, $drive->{file});
3925             push @$vollist, $drive->{file};
3926         }
3927
3928         # ignore efidisk here, already added in bios/fw handling code above
3929         return if $drive->{interface} eq 'efidisk';
3930         # similar for TPM
3931         return if $drive->{interface} eq 'tpmstate';
3932
3933         $use_virtio = 1 if $ds =~ m/^virtio/;
3934
3935         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3936
3937         if ($drive->{interface} eq 'virtio'){
3938            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3939         }
3940
3941         if ($drive->{interface} eq 'scsi') {
3942
3943             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3944
3945             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3946                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3947
3948             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3949             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3950
3951             my $iothread = '';
3952             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3953                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3954                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3955             } elsif ($drive->{iothread}) {
3956                 log_warn(
3957                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3958                 );
3959             }
3960
3961             my $queues = '';
3962             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3963                 $queues = ",num_queues=$drive->{queues}";
3964             }
3965
3966             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3967                 if !$scsicontroller->{$controller};
3968             $scsicontroller->{$controller}=1;
3969         }
3970
3971         if ($drive->{interface} eq 'sata') {
3972             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3973             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3974             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3975                 if !$ahcicontroller->{$controller};
3976             $ahcicontroller->{$controller}=1;
3977         }
3978
3979         my $pbs_conf = $pbs_backing->{$ds};
3980         my $pbs_name = undef;
3981         if ($pbs_conf) {
3982             $pbs_name = "drive-$ds-pbs";
3983             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3984         }
3985
3986         my $drive_cmd = print_drive_commandline_full(
3987             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3988
3989         # extra protection for templates, but SATA and IDE don't support it..
3990         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3991
3992         push @$devices, '-drive',$drive_cmd;
3993         push @$devices, '-device', print_drivedevice_full(
3994             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3995     });
3996
3997     for (my $i = 0; $i < $MAX_NETS; $i++) {
3998         my $netname = "net$i";
3999
4000         next if !$conf->{$netname};
4001         my $d = parse_net($conf->{$netname});
4002         next if !$d;
4003         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4004
4005         $use_virtio = 1 if $d->{model} eq 'virtio';
4006
4007         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4008
4009         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4010         push @$devices, '-netdev', $netdevfull;
4011
4012         my $netdevicefull = print_netdevice_full(
4013             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4014
4015         push @$devices, '-device', $netdevicefull;
4016     }
4017
4018     if ($conf->{ivshmem}) {
4019         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4020
4021         my $bus;
4022         if ($q35) {
4023             $bus = print_pcie_addr("ivshmem");
4024         } else {
4025             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4026         }
4027
4028         my $ivshmem_name = $ivshmem->{name} // $vmid;
4029         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4030
4031         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4032         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4033             .",size=$ivshmem->{size}M";
4034     }
4035
4036     # pci.4 is nested in pci.1
4037     $bridges->{1} = 1 if $bridges->{4};
4038
4039     if (!$q35) { # add pci bridges
4040         if (min_version($machine_version, 2, 3)) {
4041            $bridges->{1} = 1;
4042            $bridges->{2} = 1;
4043         }
4044         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4045     }
4046
4047     for my $k (sort {$b cmp $a} keys %$bridges) {
4048         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4049
4050         my $k_name = $k;
4051         if ($k == 2 && $legacy_igd) {
4052             $k_name = "$k-igd";
4053         }
4054         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4055         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4056
4057         if ($q35) { # add after -readconfig pve-q35.cfg
4058             splice @$devices, 2, 0, '-device', $devstr;
4059         } else {
4060             unshift @$devices, '-device', $devstr if $k > 0;
4061         }
4062     }
4063
4064     if (!$kvm) {
4065         push @$machineFlags, 'accel=tcg';
4066     }
4067
4068     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4069
4070     my $machine_type_min = $machine_type;
4071     if ($add_pve_version) {
4072         $machine_type_min =~ s/\+pve\d+$//;
4073         $machine_type_min .= "+pve$required_pve_version";
4074     }
4075     push @$machineFlags, "type=${machine_type_min}";
4076
4077     push @$cmd, @$devices;
4078     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4079     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4080     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4081
4082     if (my $vmstate = $conf->{vmstate}) {
4083         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4084         push @$vollist, $vmstate;
4085         push @$cmd, '-loadstate', $statepath;
4086         print "activating and using '$vmstate' as vmstate\n";
4087     }
4088
4089     if (PVE::QemuConfig->is_template($conf)) {
4090         # needed to workaround base volumes being read-only
4091         push @$cmd, '-snapshot';
4092     }
4093
4094     # add custom args
4095     if ($conf->{args}) {
4096         my $aa = PVE::Tools::split_args($conf->{args});
4097         push @$cmd, @$aa;
4098     }
4099
4100     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4101 }
4102
4103 sub check_rng_source {
4104     my ($source) = @_;
4105
4106     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4107     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4108         if ! -e $source;
4109
4110     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4111     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4112         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4113         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4114         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4115             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4116             ." to the host.\n";
4117     }
4118 }
4119
4120 sub spice_port {
4121     my ($vmid) = @_;
4122
4123     my $res = mon_cmd($vmid, 'query-spice');
4124
4125     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4126 }
4127
4128 sub vm_devices_list {
4129     my ($vmid) = @_;
4130
4131     my $res = mon_cmd($vmid, 'query-pci');
4132     my $devices_to_check = [];
4133     my $devices = {};
4134     foreach my $pcibus (@$res) {
4135         push @$devices_to_check, @{$pcibus->{devices}},
4136     }
4137
4138     while (@$devices_to_check) {
4139         my $to_check = [];
4140         for my $d (@$devices_to_check) {
4141             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4142             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4143
4144             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4145             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4146         }
4147         $devices_to_check = $to_check;
4148     }
4149
4150     my $resblock = mon_cmd($vmid, 'query-block');
4151     foreach my $block (@$resblock) {
4152         if($block->{device} =~ m/^drive-(\S+)/){
4153                 $devices->{$1} = 1;
4154         }
4155     }
4156
4157     my $resmice = mon_cmd($vmid, 'query-mice');
4158     foreach my $mice (@$resmice) {
4159         if ($mice->{name} eq 'QEMU HID Tablet') {
4160             $devices->{tablet} = 1;
4161             last;
4162         }
4163     }
4164
4165     # for usb devices there is no query-usb
4166     # but we can iterate over the entries in
4167     # qom-list path=/machine/peripheral
4168     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4169     foreach my $per (@$resperipheral) {
4170         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4171             $devices->{$per->{name}} = 1;
4172         }
4173     }
4174
4175     return $devices;
4176 }
4177
4178 sub vm_deviceplug {
4179     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4180
4181     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4182
4183     my $devices_list = vm_devices_list($vmid);
4184     return 1 if defined($devices_list->{$deviceid});
4185
4186     # add PCI bridge if we need it for the device
4187     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4188
4189     if ($deviceid eq 'tablet') {
4190         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4191     } elsif ($deviceid eq 'keyboard') {
4192         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4193     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4194         my $id = $1;
4195         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4196         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4197     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4198         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4199     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4200         qemu_iothread_add($vmid, $deviceid, $device);
4201
4202         qemu_driveadd($storecfg, $vmid, $device);
4203         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4204
4205         qemu_deviceadd($vmid, $devicefull);
4206         eval { qemu_deviceaddverify($vmid, $deviceid); };
4207         if (my $err = $@) {
4208             eval { qemu_drivedel($vmid, $deviceid); };
4209             warn $@ if $@;
4210             die $err;
4211         }
4212     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4213         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4214         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4215         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4216
4217         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4218
4219         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4220             qemu_iothread_add($vmid, $deviceid, $device);
4221             $devicefull .= ",iothread=iothread-$deviceid";
4222         }
4223
4224         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4225             $devicefull .= ",num_queues=$device->{queues}";
4226         }
4227
4228         qemu_deviceadd($vmid, $devicefull);
4229         qemu_deviceaddverify($vmid, $deviceid);
4230     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4231         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4232         qemu_driveadd($storecfg, $vmid, $device);
4233
4234         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4235         eval { qemu_deviceadd($vmid, $devicefull); };
4236         if (my $err = $@) {
4237             eval { qemu_drivedel($vmid, $deviceid); };
4238             warn $@ if $@;
4239             die $err;
4240         }
4241     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4242         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4243
4244         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4245         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4246         my $use_old_bios_files = undef;
4247         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4248
4249         my $netdevicefull = print_netdevice_full(
4250             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4251         qemu_deviceadd($vmid, $netdevicefull);
4252         eval {
4253             qemu_deviceaddverify($vmid, $deviceid);
4254             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4255         };
4256         if (my $err = $@) {
4257             eval { qemu_netdevdel($vmid, $deviceid); };
4258             warn $@ if $@;
4259             die $err;
4260         }
4261     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4262         my $bridgeid = $2;
4263         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4264         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4265
4266         qemu_deviceadd($vmid, $devicefull);
4267         qemu_deviceaddverify($vmid, $deviceid);
4268     } else {
4269         die "can't hotplug device '$deviceid'\n";
4270     }
4271
4272     return 1;
4273 }
4274
4275 # fixme: this should raise exceptions on error!
4276 sub vm_deviceunplug {
4277     my ($vmid, $conf, $deviceid) = @_;
4278
4279     my $devices_list = vm_devices_list($vmid);
4280     return 1 if !defined($devices_list->{$deviceid});
4281
4282     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4283     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4284
4285     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4286         qemu_devicedel($vmid, $deviceid);
4287     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4288         qemu_devicedel($vmid, $deviceid);
4289         qemu_devicedelverify($vmid, $deviceid);
4290     } elsif ($deviceid =~ m/^usb\d+$/) {
4291         qemu_devicedel($vmid, $deviceid);
4292         qemu_devicedelverify($vmid, $deviceid);
4293     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4294         my $device = parse_drive($deviceid, $conf->{$deviceid});
4295
4296         qemu_devicedel($vmid, $deviceid);
4297         qemu_devicedelverify($vmid, $deviceid);
4298         qemu_drivedel($vmid, $deviceid);
4299         qemu_iothread_del($vmid, $deviceid, $device);
4300     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4301         qemu_devicedel($vmid, $deviceid);
4302         qemu_devicedelverify($vmid, $deviceid);
4303     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4304         my $device = parse_drive($deviceid, $conf->{$deviceid});
4305
4306         qemu_devicedel($vmid, $deviceid);
4307         qemu_devicedelverify($vmid, $deviceid);
4308         qemu_drivedel($vmid, $deviceid);
4309         qemu_deletescsihw($conf, $vmid, $deviceid);
4310
4311         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4312             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4313     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4314         qemu_devicedel($vmid, $deviceid);
4315         qemu_devicedelverify($vmid, $deviceid);
4316         qemu_netdevdel($vmid, $deviceid);
4317     } else {
4318         die "can't unplug device '$deviceid'\n";
4319     }
4320
4321     return 1;
4322 }
4323
4324 sub qemu_spice_usbredir_chardev_add {
4325     my ($vmid, $id) = @_;
4326
4327     mon_cmd($vmid, "chardev-add" , (
4328         id => $id,
4329         backend => {
4330             type => 'spicevmc',
4331             data => {
4332                 type => "usbredir",
4333             },
4334         },
4335     ));
4336 }
4337
4338 sub qemu_iothread_add {
4339     my ($vmid, $deviceid, $device) = @_;
4340
4341     if ($device->{iothread}) {
4342         my $iothreads = vm_iothreads_list($vmid);
4343         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4344     }
4345 }
4346
4347 sub qemu_iothread_del {
4348     my ($vmid, $deviceid, $device) = @_;
4349
4350     if ($device->{iothread}) {
4351         my $iothreads = vm_iothreads_list($vmid);
4352         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4353     }
4354 }
4355
4356 sub qemu_driveadd {
4357     my ($storecfg, $vmid, $device) = @_;
4358
4359     my $kvmver = get_running_qemu_version($vmid);
4360     my $io_uring = min_version($kvmver, 6, 0);
4361     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4362     $drive =~ s/\\/\\\\/g;
4363     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4364
4365     # If the command succeeds qemu prints: "OK"
4366     return 1 if $ret =~ m/OK/s;
4367
4368     die "adding drive failed: $ret\n";
4369 }
4370
4371 sub qemu_drivedel {
4372     my ($vmid, $deviceid) = @_;
4373
4374     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4375     $ret =~ s/^\s+//;
4376
4377     return 1 if $ret eq "";
4378
4379     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4380     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4381
4382     die "deleting drive $deviceid failed : $ret\n";
4383 }
4384
4385 sub qemu_deviceaddverify {
4386     my ($vmid, $deviceid) = @_;
4387
4388     for (my $i = 0; $i <= 5; $i++) {
4389          my $devices_list = vm_devices_list($vmid);
4390          return 1 if defined($devices_list->{$deviceid});
4391          sleep 1;
4392     }
4393
4394     die "error on hotplug device '$deviceid'\n";
4395 }
4396
4397
4398 sub qemu_devicedelverify {
4399     my ($vmid, $deviceid) = @_;
4400
4401     # need to verify that the device is correctly removed as device_del
4402     # is async and empty return is not reliable
4403
4404     for (my $i = 0; $i <= 5; $i++) {
4405          my $devices_list = vm_devices_list($vmid);
4406          return 1 if !defined($devices_list->{$deviceid});
4407          sleep 1;
4408     }
4409
4410     die "error on hot-unplugging device '$deviceid'\n";
4411 }
4412
4413 sub qemu_findorcreatescsihw {
4414     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4415
4416     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4417
4418     my $scsihwid="$controller_prefix$controller";
4419     my $devices_list = vm_devices_list($vmid);
4420
4421     if (!defined($devices_list->{$scsihwid})) {
4422         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4423     }
4424
4425     return 1;
4426 }
4427
4428 sub qemu_deletescsihw {
4429     my ($conf, $vmid, $opt) = @_;
4430
4431     my $device = parse_drive($opt, $conf->{$opt});
4432
4433     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4434         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4435         return 1;
4436     }
4437
4438     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4439
4440     my $devices_list = vm_devices_list($vmid);
4441     foreach my $opt (keys %{$devices_list}) {
4442         if (is_valid_drivename($opt)) {
4443             my $drive = parse_drive($opt, $conf->{$opt});
4444             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4445                 return 1;
4446             }
4447         }
4448     }
4449
4450     my $scsihwid="scsihw$controller";
4451
4452     vm_deviceunplug($vmid, $conf, $scsihwid);
4453
4454     return 1;
4455 }
4456
4457 sub qemu_add_pci_bridge {
4458     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4459
4460     my $bridges = {};
4461
4462     my $bridgeid;
4463
4464     print_pci_addr($device, $bridges, $arch, $machine_type);
4465
4466     while (my ($k, $v) = each %$bridges) {
4467         $bridgeid = $k;
4468     }
4469     return 1 if !defined($bridgeid) || $bridgeid < 1;
4470
4471     my $bridge = "pci.$bridgeid";
4472     my $devices_list = vm_devices_list($vmid);
4473
4474     if (!defined($devices_list->{$bridge})) {
4475         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4476     }
4477
4478     return 1;
4479 }
4480
4481 sub qemu_set_link_status {
4482     my ($vmid, $device, $up) = @_;
4483
4484     mon_cmd($vmid, "set_link", name => $device,
4485                up => $up ? JSON::true : JSON::false);
4486 }
4487
4488 sub qemu_netdevadd {
4489     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4490
4491     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4492     my %options =  split(/[=,]/, $netdev);
4493
4494     if (defined(my $vhost = $options{vhost})) {
4495         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4496     }
4497
4498     if (defined(my $queues = $options{queues})) {
4499         $options{queues} = $queues + 0;
4500     }
4501
4502     mon_cmd($vmid, "netdev_add",  %options);
4503     return 1;
4504 }
4505
4506 sub qemu_netdevdel {
4507     my ($vmid, $deviceid) = @_;
4508
4509     mon_cmd($vmid, "netdev_del", id => $deviceid);
4510 }
4511
4512 sub qemu_usb_hotplug {
4513     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4514
4515     return if !$device;
4516
4517     # remove the old one first
4518     vm_deviceunplug($vmid, $conf, $deviceid);
4519
4520     # check if xhci controller is necessary and available
4521     my $devicelist = vm_devices_list($vmid);
4522
4523     if (!$devicelist->{xhci}) {
4524         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4525         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4526     }
4527
4528     # add the new one
4529     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4530 }
4531
4532 sub qemu_cpu_hotplug {
4533     my ($vmid, $conf, $vcpus) = @_;
4534
4535     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4536
4537     my $sockets = 1;
4538     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4539     $sockets = $conf->{sockets} if  $conf->{sockets};
4540     my $cores = $conf->{cores} || 1;
4541     my $maxcpus = $sockets * $cores;
4542
4543     $vcpus = $maxcpus if !$vcpus;
4544
4545     die "you can't add more vcpus than maxcpus\n"
4546         if $vcpus > $maxcpus;
4547
4548     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4549
4550     if ($vcpus < $currentvcpus) {
4551
4552         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4553
4554             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4555                 qemu_devicedel($vmid, "cpu$i");
4556                 my $retry = 0;
4557                 my $currentrunningvcpus = undef;
4558                 while (1) {
4559                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4560                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4561                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4562                     $retry++;
4563                     sleep 1;
4564                 }
4565                 #update conf after each succesfull cpu unplug
4566                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4567                 PVE::QemuConfig->write_config($vmid, $conf);
4568             }
4569         } else {
4570             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4571         }
4572
4573         return;
4574     }
4575
4576     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4577     die "vcpus in running vm does not match its configuration\n"
4578         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4579
4580     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4581
4582         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4583             my $cpustr = print_cpu_device($conf, $i);
4584             qemu_deviceadd($vmid, $cpustr);
4585
4586             my $retry = 0;
4587             my $currentrunningvcpus = undef;
4588             while (1) {
4589                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4590                 last if scalar(@{$currentrunningvcpus}) == $i;
4591                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4592                 sleep 1;
4593                 $retry++;
4594             }
4595             #update conf after each succesfull cpu hotplug
4596             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4597             PVE::QemuConfig->write_config($vmid, $conf);
4598         }
4599     } else {
4600
4601         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4602             mon_cmd($vmid, "cpu-add", id => int($i));
4603         }
4604     }
4605 }
4606
4607 sub qemu_block_set_io_throttle {
4608     my ($vmid, $deviceid,
4609         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4610         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4611         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4612         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4613
4614     return if !check_running($vmid) ;
4615
4616     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4617         bps => int($bps),
4618         bps_rd => int($bps_rd),
4619         bps_wr => int($bps_wr),
4620         iops => int($iops),
4621         iops_rd => int($iops_rd),
4622         iops_wr => int($iops_wr),
4623         bps_max => int($bps_max),
4624         bps_rd_max => int($bps_rd_max),
4625         bps_wr_max => int($bps_wr_max),
4626         iops_max => int($iops_max),
4627         iops_rd_max => int($iops_rd_max),
4628         iops_wr_max => int($iops_wr_max),
4629         bps_max_length => int($bps_max_length),
4630         bps_rd_max_length => int($bps_rd_max_length),
4631         bps_wr_max_length => int($bps_wr_max_length),
4632         iops_max_length => int($iops_max_length),
4633         iops_rd_max_length => int($iops_rd_max_length),
4634         iops_wr_max_length => int($iops_wr_max_length),
4635     );
4636
4637 }
4638
4639 sub qemu_block_resize {
4640     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4641
4642     my $running = check_running($vmid);
4643
4644     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4645
4646     return if !$running;
4647
4648     my $padding = (1024 - $size % 1024) % 1024;
4649     $size = $size + $padding;
4650
4651     mon_cmd(
4652         $vmid,
4653         "block_resize",
4654         device => $deviceid,
4655         size => int($size),
4656         timeout => 60,
4657     );
4658 }
4659
4660 sub qemu_volume_snapshot {
4661     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4662
4663     my $running = check_running($vmid);
4664
4665     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4666         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4667     } else {
4668         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4669     }
4670 }
4671
4672 sub qemu_volume_snapshot_delete {
4673     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4674
4675     my $running = check_running($vmid);
4676
4677     if($running) {
4678
4679         $running = undef;
4680         my $conf = PVE::QemuConfig->load_config($vmid);
4681         PVE::QemuConfig->foreach_volume($conf, sub {
4682             my ($ds, $drive) = @_;
4683             $running = 1 if $drive->{file} eq $volid;
4684         });
4685     }
4686
4687     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4688         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4689     } else {
4690         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4691     }
4692 }
4693
4694 sub set_migration_caps {
4695     my ($vmid, $savevm) = @_;
4696
4697     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4698
4699     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4700     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4701
4702     my $cap_ref = [];
4703
4704     my $enabled_cap = {
4705         "auto-converge" => 1,
4706         "xbzrle" => 1,
4707         "x-rdma-pin-all" => 0,
4708         "zero-blocks" => 0,
4709         "compress" => 0,
4710         "dirty-bitmaps" => $dirty_bitmaps,
4711     };
4712
4713     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4714
4715     for my $supported_capability (@$supported_capabilities) {
4716         push @$cap_ref, {
4717             capability => $supported_capability->{capability},
4718             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4719         };
4720     }
4721
4722     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4723 }
4724
4725 sub foreach_volid {
4726     my ($conf, $func, @param) = @_;
4727
4728     my $volhash = {};
4729
4730     my $test_volid = sub {
4731         my ($key, $drive, $snapname, $pending) = @_;
4732
4733         my $volid = $drive->{file};
4734         return if !$volid;
4735
4736         $volhash->{$volid}->{cdrom} //= 1;
4737         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4738
4739         my $replicate = $drive->{replicate} // 1;
4740         $volhash->{$volid}->{replicate} //= 0;
4741         $volhash->{$volid}->{replicate} = 1 if $replicate;
4742
4743         $volhash->{$volid}->{shared} //= 0;
4744         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4745
4746         $volhash->{$volid}->{is_unused} //= 0;
4747         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4748
4749         $volhash->{$volid}->{is_attached} //= 0;
4750         $volhash->{$volid}->{is_attached} = 1
4751             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4752
4753         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4754             if defined($snapname);
4755
4756         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4757
4758         my $size = $drive->{size};
4759         $volhash->{$volid}->{size} //= $size if $size;
4760
4761         $volhash->{$volid}->{is_vmstate} //= 0;
4762         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4763
4764         $volhash->{$volid}->{is_tpmstate} //= 0;
4765         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4766
4767         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4768     };
4769
4770     my $include_opts = {
4771         extra_keys => ['vmstate'],
4772         include_unused => 1,
4773     };
4774
4775     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4776
4777     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4778         if defined($conf->{pending}) && $conf->{pending}->%*;
4779
4780     foreach my $snapname (keys %{$conf->{snapshots}}) {
4781         my $snap = $conf->{snapshots}->{$snapname};
4782         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4783     }
4784
4785     foreach my $volid (keys %$volhash) {
4786         &$func($volid, $volhash->{$volid}, @param);
4787     }
4788 }
4789
4790 my $fast_plug_option = {
4791     'description' => 1,
4792     'hookscript' => 1,
4793     'lock' => 1,
4794     'migrate_downtime' => 1,
4795     'migrate_speed' => 1,
4796     'name' => 1,
4797     'onboot' => 1,
4798     'protection' => 1,
4799     'shares' => 1,
4800     'startup' => 1,
4801     'tags' => 1,
4802     'vmstatestorage' => 1,
4803 };
4804
4805 for my $opt (keys %$confdesc_cloudinit) {
4806     $fast_plug_option->{$opt} = 1;
4807 };
4808
4809 # hotplug changes in [PENDING]
4810 # $selection hash can be used to only apply specified options, for
4811 # example: { cores => 1 } (only apply changed 'cores')
4812 # $errors ref is used to return error messages
4813 sub vmconfig_hotplug_pending {
4814     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4815
4816     my $defaults = load_defaults();
4817     my $arch = get_vm_arch($conf);
4818     my $machine_type = get_vm_machine($conf, undef, $arch);
4819
4820     # commit values which do not have any impact on running VM first
4821     # Note: those option cannot raise errors, we we do not care about
4822     # $selection and always apply them.
4823
4824     my $add_error = sub {
4825         my ($opt, $msg) = @_;
4826         $errors->{$opt} = "hotplug problem - $msg";
4827     };
4828
4829     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4830
4831     my $cloudinit_record_changed = sub {
4832         my ($conf, $opt, $old, $new) = @_;
4833         return if !$cloudinit_pending_properties->{$opt};
4834
4835         my $ci = ($conf->{cloudinit} //= {});
4836
4837         my $recorded = $ci->{$opt};
4838         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4839
4840         if (defined($new)) {
4841             if (defined($old)) {
4842                 # an existing value is being modified
4843                 if (defined($recorded)) {
4844                     # the value was already not in sync
4845                     if ($new eq $recorded) {
4846                         # a value is being reverted to the cloud-init state:
4847                         delete $ci->{$opt};
4848                         delete $added{$opt};
4849                     } else {
4850                         # the value was changed multiple times, do nothing
4851                     }
4852                 } elsif ($added{$opt}) {
4853                     # the value had been marked as added and is being changed, do nothing
4854                 } else {
4855                     # the value is new, record it:
4856                     $ci->{$opt} = $old;
4857                 }
4858             } else {
4859                 # a new value is being added
4860                 if (defined($recorded)) {
4861                     # it was already not in sync
4862                     if ($new eq $recorded) {
4863                         # a value is being reverted to the cloud-init state:
4864                         delete $ci->{$opt};
4865                         delete $added{$opt};
4866                     } else {
4867                         # the value had temporarily been removed, do nothing
4868                     }
4869                 } elsif ($added{$opt}) {
4870                     # the value had been marked as added already, do nothing
4871                 } else {
4872                     # the value is new, add it
4873                     $added{$opt} = 1;
4874                 }
4875             }
4876         } elsif (!defined($old)) {
4877             # a non-existent value is being removed? ignore...
4878         } else {
4879             # a value is being deleted
4880             if (defined($recorded)) {
4881                 # a value was already recorded, just keep it
4882             } elsif ($added{$opt}) {
4883                 # the value was marked as added, remove it
4884                 delete $added{$opt};
4885             } else {
4886                 # a previously unrecorded value is being removed, record the old value:
4887                 $ci->{$opt} = $old;
4888             }
4889         }
4890
4891         my $added = join(',', sort keys %added);
4892         $ci->{added} = $added if length($added);
4893     };
4894
4895     my $changes = 0;
4896     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4897         if ($fast_plug_option->{$opt}) {
4898             my $new = delete $conf->{pending}->{$opt};
4899             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4900             $conf->{$opt} = $new;
4901             $changes = 1;
4902         }
4903     }
4904
4905     if ($changes) {
4906         PVE::QemuConfig->write_config($vmid, $conf);
4907     }
4908
4909     my $ostype = $conf->{ostype};
4910     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4911     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4912     my $usb_hotplug = $hotplug_features->{usb}
4913         && min_version($version, 7, 1)
4914         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4915
4916     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4917     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4918
4919     foreach my $opt (sort keys %$pending_delete_hash) {
4920         next if $selection && !$selection->{$opt};
4921         my $force = $pending_delete_hash->{$opt}->{force};
4922         eval {
4923             if ($opt eq 'hotplug') {
4924                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4925             } elsif ($opt eq 'tablet') {
4926                 die "skip\n" if !$hotplug_features->{usb};
4927                 if ($defaults->{tablet}) {
4928                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4929                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4930                         if $arch eq 'aarch64';
4931                 } else {
4932                     vm_deviceunplug($vmid, $conf, 'tablet');
4933                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4934                 }
4935             } elsif ($opt =~ m/^usb(\d+)$/) {
4936                 my $index = $1;
4937                 die "skip\n" if !$usb_hotplug;
4938                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4939                 vm_deviceunplug($vmid, $conf, $opt);
4940             } elsif ($opt eq 'vcpus') {
4941                 die "skip\n" if !$hotplug_features->{cpu};
4942                 qemu_cpu_hotplug($vmid, $conf, undef);
4943             } elsif ($opt eq 'balloon') {
4944                 # enable balloon device is not hotpluggable
4945                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4946                 # here we reset the ballooning value to memory
4947                 my $balloon = get_current_memory($conf->{memory});
4948                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4949             } elsif ($fast_plug_option->{$opt}) {
4950                 # do nothing
4951             } elsif ($opt =~ m/^net(\d+)$/) {
4952                 die "skip\n" if !$hotplug_features->{network};
4953                 vm_deviceunplug($vmid, $conf, $opt);
4954                 if($have_sdn) {
4955                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
4956                     PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
4957                 }
4958             } elsif (is_valid_drivename($opt)) {
4959                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4960                 vm_deviceunplug($vmid, $conf, $opt);
4961                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4962             } elsif ($opt =~ m/^memory$/) {
4963                 die "skip\n" if !$hotplug_features->{memory};
4964                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
4965             } elsif ($opt eq 'cpuunits') {
4966                 $cgroup->change_cpu_shares(undef);
4967             } elsif ($opt eq 'cpulimit') {
4968                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4969             } else {
4970                 die "skip\n";
4971             }
4972         };
4973         if (my $err = $@) {
4974             &$add_error($opt, $err) if $err ne "skip\n";
4975         } else {
4976             my $old = delete $conf->{$opt};
4977             $cloudinit_record_changed->($conf, $opt, $old, undef);
4978             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4979         }
4980     }
4981
4982     my $cloudinit_opt;
4983     foreach my $opt (keys %{$conf->{pending}}) {
4984         next if $selection && !$selection->{$opt};
4985         my $value = $conf->{pending}->{$opt};
4986         eval {
4987             if ($opt eq 'hotplug') {
4988                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4989                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
4990             } elsif ($opt eq 'tablet') {
4991                 die "skip\n" if !$hotplug_features->{usb};
4992                 if ($value == 1) {
4993                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4994                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4995                         if $arch eq 'aarch64';
4996                 } elsif ($value == 0) {
4997                     vm_deviceunplug($vmid, $conf, 'tablet');
4998                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4999                 }
5000             } elsif ($opt =~ m/^usb(\d+)$/) {
5001                 my $index = $1;
5002                 die "skip\n" if !$usb_hotplug;
5003                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5004                 my $id = $opt;
5005                 if ($d->{host} =~ m/^spice$/i)  {
5006                     $id = "usbredirdev$index";
5007                 }
5008                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5009             } elsif ($opt eq 'vcpus') {
5010                 die "skip\n" if !$hotplug_features->{cpu};
5011                 qemu_cpu_hotplug($vmid, $conf, $value);
5012             } elsif ($opt eq 'balloon') {
5013                 # enable/disable balloning device is not hotpluggable
5014                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5015                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5016                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5017
5018                 # allow manual ballooning if shares is set to zero
5019                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5020                     my $memory = get_current_memory($conf->{memory});
5021                     my $balloon = $conf->{pending}->{balloon} || $memory;
5022                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5023                 }
5024             } elsif ($opt =~ m/^net(\d+)$/) {
5025                 # some changes can be done without hotplug
5026                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5027                                     $vmid, $opt, $value, $arch, $machine_type);
5028             } elsif (is_valid_drivename($opt)) {
5029                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5030                 # some changes can be done without hotplug
5031                 my $drive = parse_drive($opt, $value);
5032                 if (drive_is_cloudinit($drive)) {
5033                     $cloudinit_opt = [$opt, $drive];
5034                     # apply all the other changes first, then generate the cloudinit disk
5035                     die "skip\n";
5036                 }
5037                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5038                                      $vmid, $opt, $value, $arch, $machine_type);
5039             } elsif ($opt =~ m/^memory$/) { #dimms
5040                 die "skip\n" if !$hotplug_features->{memory};
5041                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5042             } elsif ($opt eq 'cpuunits') {
5043                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5044                 $cgroup->change_cpu_shares($new_cpuunits);
5045             } elsif ($opt eq 'cpulimit') {
5046                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5047                 $cgroup->change_cpu_quota($cpulimit, 100000);
5048             } elsif ($opt eq 'agent') {
5049                 vmconfig_update_agent($conf, $opt, $value);
5050             } else {
5051                 die "skip\n";  # skip non-hot-pluggable options
5052             }
5053         };
5054         if (my $err = $@) {
5055             &$add_error($opt, $err) if $err ne "skip\n";
5056         } else {
5057             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5058             $conf->{$opt} = $value;
5059             delete $conf->{pending}->{$opt};
5060         }
5061     }
5062
5063     if (defined($cloudinit_opt)) {
5064         my ($opt, $drive) = @$cloudinit_opt;
5065         my $value = $conf->{pending}->{$opt};
5066         eval {
5067             my $temp = {%$conf, $opt => $value};
5068             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5069             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5070                                  $vmid, $opt, $value, $arch, $machine_type);
5071         };
5072         if (my $err = $@) {
5073             &$add_error($opt, $err) if $err ne "skip\n";
5074         } else {
5075             $conf->{$opt} = $value;
5076             delete $conf->{pending}->{$opt};
5077         }
5078     }
5079
5080     # unplug xhci controller if no usb device is left
5081     if ($usb_hotplug) {
5082         my $has_usb = 0;
5083         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5084             next if !defined($conf->{"usb$i"});
5085             $has_usb = 1;
5086             last;
5087         }
5088         if (!$has_usb) {
5089             vm_deviceunplug($vmid, $conf, 'xhci');
5090         }
5091     }
5092
5093     PVE::QemuConfig->write_config($vmid, $conf);
5094
5095     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5096         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5097     }
5098 }
5099
5100 sub try_deallocate_drive {
5101     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5102
5103     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5104         my $volid = $drive->{file};
5105         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5106             my $sid = PVE::Storage::parse_volume_id($volid);
5107             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5108
5109             # check if the disk is really unused
5110             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5111                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5112             PVE::Storage::vdisk_free($storecfg, $volid);
5113             return 1;
5114         } else {
5115             # If vm is not owner of this disk remove from config
5116             return 1;
5117         }
5118     }
5119
5120     return;
5121 }
5122
5123 sub vmconfig_delete_or_detach_drive {
5124     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5125
5126     my $drive = parse_drive($opt, $conf->{$opt});
5127
5128     my $rpcenv = PVE::RPCEnvironment::get();
5129     my $authuser = $rpcenv->get_user();
5130
5131     if ($force) {
5132         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5133         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5134     } else {
5135         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5136     }
5137 }
5138
5139
5140
5141 sub vmconfig_apply_pending {
5142     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5143
5144     return if !scalar(keys %{$conf->{pending}});
5145
5146     my $add_apply_error = sub {
5147         my ($opt, $msg) = @_;
5148         my $err_msg = "unable to apply pending change $opt : $msg";
5149         $errors->{$opt} = $err_msg;
5150         warn $err_msg;
5151     };
5152
5153     # cold plug
5154
5155     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5156     foreach my $opt (sort keys %$pending_delete_hash) {
5157         my $force = $pending_delete_hash->{$opt}->{force};
5158         eval {
5159             if ($opt =~ m/^unused/) {
5160                 die "internal error";
5161             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5162                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5163             } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5164                 if($have_sdn) {
5165                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5166                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5167                     warn if $@;
5168                 }
5169             }
5170         };
5171         if (my $err = $@) {
5172             $add_apply_error->($opt, $err);
5173         } else {
5174             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5175             delete $conf->{$opt};
5176         }
5177     }
5178
5179     PVE::QemuConfig->cleanup_pending($conf);
5180
5181     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5182
5183     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5184         next if $opt eq 'delete'; # just to be sure
5185         eval {
5186             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5187                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5188             } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5189                 if($have_sdn) {
5190                     my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5191                     if ($conf->{$opt}){
5192                         my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5193
5194                         if ($old_net->{bridge} ne $new_net->{bridge} ||
5195                             $old_net->{macaddr} ne $new_net->{macaddr}) {
5196                             PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5197                         }
5198                    }
5199                    #fixme: reuse ip if mac change && same bridge
5200                    PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5201                 }
5202             }
5203         };
5204         if (my $err = $@) {
5205             $add_apply_error->($opt, $err);
5206         } else {
5207
5208             if (is_valid_drivename($opt)) {
5209                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5210                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5211             }
5212
5213             $conf->{$opt} = delete $conf->{pending}->{$opt};
5214         }
5215     }
5216
5217     # write all changes at once to avoid unnecessary i/o
5218     PVE::QemuConfig->write_config($vmid, $conf);
5219     if ($generate_cloudinit) {
5220         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5221             # After successful generation and if there were changes to be applied, update the
5222             # config to drop the {cloudinit} entry.
5223             PVE::QemuConfig->write_config($vmid, $conf);
5224         }
5225     }
5226 }
5227
5228 sub vmconfig_update_net {
5229     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5230
5231     my $newnet = parse_net($value);
5232
5233     if ($conf->{$opt}) {
5234         my $oldnet = parse_net($conf->{$opt});
5235
5236         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5237             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5238             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5239             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5240             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5241
5242             # for non online change, we try to hot-unplug
5243             die "skip\n" if !$hotplug;
5244             vm_deviceunplug($vmid, $conf, $opt);
5245
5246             if($have_sdn) {
5247                 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5248             }
5249
5250         } else {
5251
5252             die "internal error" if $opt !~ m/net(\d+)/;
5253             my $iface = "tap${vmid}i$1";
5254
5255             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5256                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5257                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5258                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5259                 PVE::Network::tap_unplug($iface);
5260
5261                 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5262                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5263                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5264                     qemu_set_link_status($vmid, $opt, 0);
5265                 }
5266
5267                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5268                     if ($have_sdn) {
5269                         PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5270                         PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5271                     }
5272                 }
5273
5274                 if ($have_sdn) {
5275                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5276                 } else {
5277                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5278                 }
5279
5280                 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5281                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5282                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5283                     qemu_set_link_status($vmid, $opt, 1);
5284                 }
5285
5286             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5287                 # Rate can be applied on its own but any change above needs to
5288                 # include the rate in tap_plug since OVS resets everything.
5289                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5290             }
5291
5292             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5293                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5294             }
5295
5296             return 1;
5297         }
5298     }
5299
5300     if ($hotplug) {
5301         if ($have_sdn) {
5302             PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5303             PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5304         }
5305         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5306     } else {
5307         die "skip\n";
5308     }
5309 }
5310
5311 sub vmconfig_update_agent {
5312     my ($conf, $opt, $value) = @_;
5313
5314     die "skip\n" if !$conf->{$opt};
5315
5316     my $hotplug_options = { fstrim_cloned_disks => 1 };
5317
5318     my $old_agent = parse_guest_agent($conf);
5319     my $agent = parse_guest_agent({$opt => $value});
5320
5321     for my $option (keys %$agent) { # added/changed options
5322         next if defined($hotplug_options->{$option});
5323         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5324     }
5325
5326     for my $option (keys %$old_agent) { # removed options
5327         next if defined($hotplug_options->{$option});
5328         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5329     }
5330
5331     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5332 }
5333
5334 sub vmconfig_update_disk {
5335     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5336
5337     my $drive = parse_drive($opt, $value);
5338
5339     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5340         my $media = $drive->{media} || 'disk';
5341         my $oldmedia = $old_drive->{media} || 'disk';
5342         die "unable to change media type\n" if $media ne $oldmedia;
5343
5344         if (!drive_is_cdrom($old_drive)) {
5345
5346             if ($drive->{file} ne $old_drive->{file}) {
5347
5348                 die "skip\n" if !$hotplug;
5349
5350                 # unplug and register as unused
5351                 vm_deviceunplug($vmid, $conf, $opt);
5352                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5353
5354             } else {
5355                 # update existing disk
5356
5357                 # skip non hotpluggable value
5358                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5359                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5360                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5361                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5362                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5363                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5364                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5365                     die "skip\n";
5366                 }
5367
5368                 # apply throttle
5369                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5370                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5371                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5372                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5373                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5374                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5375                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5376                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5377                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5378                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5379                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5380                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5381                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5382                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5383                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5384                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5385                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5386                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5387
5388                     qemu_block_set_io_throttle(
5389                         $vmid,"drive-$opt",
5390                         ($drive->{mbps} || 0)*1024*1024,
5391                         ($drive->{mbps_rd} || 0)*1024*1024,
5392                         ($drive->{mbps_wr} || 0)*1024*1024,
5393                          $drive->{iops} || 0,
5394                          $drive->{iops_rd} || 0,
5395                          $drive->{iops_wr} || 0,
5396                         ($drive->{mbps_max} || 0)*1024*1024,
5397                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5398                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5399                          $drive->{iops_max} || 0,
5400                          $drive->{iops_rd_max} || 0,
5401                          $drive->{iops_wr_max} || 0,
5402                          $drive->{bps_max_length} || 1,
5403                          $drive->{bps_rd_max_length} || 1,
5404                          $drive->{bps_wr_max_length} || 1,
5405                          $drive->{iops_max_length} || 1,
5406                          $drive->{iops_rd_max_length} || 1,
5407                          $drive->{iops_wr_max_length} || 1,
5408                     );
5409
5410                 }
5411
5412                 return 1;
5413             }
5414
5415         } else { # cdrom
5416
5417             if ($drive->{file} eq 'none') {
5418                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5419                 if (drive_is_cloudinit($old_drive)) {
5420                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5421                 }
5422             } else {
5423                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5424
5425                 # force eject if locked
5426                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5427
5428                 if ($path) {
5429                     mon_cmd($vmid, "blockdev-change-medium",
5430                         id => "$opt", filename => "$path");
5431                 }
5432             }
5433
5434             return 1;
5435         }
5436     }
5437
5438     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5439     # hotplug new disks
5440     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5441     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5442 }
5443
5444 sub vmconfig_update_cloudinit_drive {
5445     my ($storecfg, $conf, $vmid) = @_;
5446
5447     my $cloudinit_ds = undef;
5448     my $cloudinit_drive = undef;
5449
5450     PVE::QemuConfig->foreach_volume($conf, sub {
5451         my ($ds, $drive) = @_;
5452         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5453             $cloudinit_ds = $ds;
5454             $cloudinit_drive = $drive;
5455         }
5456     });
5457
5458     return if !$cloudinit_drive;
5459
5460     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5461         PVE::QemuConfig->write_config($vmid, $conf);
5462     }
5463
5464     my $running = PVE::QemuServer::check_running($vmid);
5465
5466     if ($running) {
5467         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5468         if ($path) {
5469             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5470             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5471         }
5472     }
5473 }
5474
5475 # called in locked context by incoming migration
5476 sub vm_migrate_get_nbd_disks {
5477     my ($storecfg, $conf, $replicated_volumes) = @_;
5478
5479     my $local_volumes = {};
5480     PVE::QemuConfig->foreach_volume($conf, sub {
5481         my ($ds, $drive) = @_;
5482
5483         return if drive_is_cdrom($drive);
5484         return if $ds eq 'tpmstate0';
5485
5486         my $volid = $drive->{file};
5487
5488         return if !$volid;
5489
5490         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5491
5492         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5493         return if $scfg->{shared};
5494
5495         my $format = qemu_img_format($scfg, $volname);
5496
5497         # replicated disks re-use existing state via bitmap
5498         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5499         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5500     });
5501     return $local_volumes;
5502 }
5503
5504 # called in locked context by incoming migration
5505 sub vm_migrate_alloc_nbd_disks {
5506     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5507
5508     my $nbd = {};
5509     foreach my $opt (sort keys %$source_volumes) {
5510         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5511
5512         if ($use_existing) {
5513             $nbd->{$opt}->{drivestr} = print_drive($drive);
5514             $nbd->{$opt}->{volid} = $volid;
5515             $nbd->{$opt}->{replicated} = 1;
5516             next;
5517         }
5518
5519         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5520
5521         # order of precedence, filtered by whether storage supports it:
5522         # 1. explicit requested format
5523         # 2. default format of storage
5524         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5525         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5526
5527         my $size = $drive->{size} / 1024;
5528         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5529         my $newdrive = $drive;
5530         $newdrive->{format} = $format;
5531         $newdrive->{file} = $newvolid;
5532         my $drivestr = print_drive($newdrive);
5533         $nbd->{$opt}->{drivestr} = $drivestr;
5534         $nbd->{$opt}->{volid} = $newvolid;
5535     }
5536
5537     return $nbd;
5538 }
5539
5540 # see vm_start_nolock for parameters, additionally:
5541 # migrate_opts:
5542 #   storagemap = parsed storage map for allocating NBD disks
5543 sub vm_start {
5544     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5545
5546     return PVE::QemuConfig->lock_config($vmid, sub {
5547         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5548
5549         die "you can't start a vm if it's a template\n"
5550             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5551
5552         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5553         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5554
5555         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5556
5557         if ($has_backup_lock && $running) {
5558             # a backup is currently running, attempt to start the guest in the
5559             # existing QEMU instance
5560             return vm_resume($vmid);
5561         }
5562
5563         PVE::QemuConfig->check_lock($conf)
5564             if !($params->{skiplock} || $has_suspended_lock);
5565
5566         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5567
5568         die "VM $vmid already running\n" if $running;
5569
5570         if (my $storagemap = $migrate_opts->{storagemap}) {
5571             my $replicated = $migrate_opts->{replicated_volumes};
5572             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5573             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5574
5575             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5576                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5577             }
5578         }
5579
5580         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5581     });
5582 }
5583
5584
5585 # params:
5586 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5587 #   skiplock => 0/1, skip checking for config lock
5588 #   skiptemplate => 0/1, skip checking whether VM is template
5589 #   forcemachine => to force QEMU machine (rollback/migration)
5590 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5591 #   timeout => in seconds
5592 #   paused => start VM in paused state (backup)
5593 #   resume => resume from hibernation
5594 #   pbs-backing => {
5595 #      sata0 => {
5596 #         repository
5597 #         snapshot
5598 #         keyfile
5599 #         archive
5600 #      },
5601 #      virtio2 => ...
5602 #   }
5603 # migrate_opts:
5604 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5605 #   migratedfrom => source node
5606 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5607 #   network => CIDR of migration network
5608 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5609 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5610 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5611 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5612 #       contained in config
5613 sub vm_start_nolock {
5614     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5615
5616     my $statefile = $params->{statefile};
5617     my $resume = $params->{resume};
5618
5619     my $migratedfrom = $migrate_opts->{migratedfrom};
5620     my $migration_type = $migrate_opts->{type};
5621
5622     my $res = {};
5623
5624     # clean up leftover reboot request files
5625     eval { clear_reboot_request($vmid); };
5626     warn $@ if $@;
5627
5628     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5629         vmconfig_apply_pending($vmid, $conf, $storecfg);
5630         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5631     }
5632
5633     # don't regenerate the ISO if the VM is started as part of a live migration
5634     # this way we can reuse the old ISO with the correct config
5635     if (!$migratedfrom) {
5636         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5637             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5638             # $conf->{cloudinit}, so we could just not do this?
5639             # But we do it above, so for now let's be consistent.
5640             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5641         }
5642     }
5643
5644     # override offline migrated volumes, conf is out of date still
5645     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5646         for my $key (sort keys $offline_volumes->%*) {
5647             my $parsed = parse_drive($key, $conf->{$key});
5648             $parsed->{file} = $offline_volumes->{$key};
5649             $conf->{$key} = print_drive($parsed);
5650         }
5651     }
5652
5653     my $defaults = load_defaults();
5654
5655     # set environment variable useful inside network script
5656     # for remote migration the config is available on the target node!
5657     if (!$migrate_opts->{remote_node}) {
5658         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5659     }
5660
5661     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5662
5663     my $forcemachine = $params->{forcemachine};
5664     my $forcecpu = $params->{forcecpu};
5665     if ($resume) {
5666         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5667         $forcemachine = $conf->{runningmachine};
5668         $forcecpu = $conf->{runningcpu};
5669         print "Resuming suspended VM\n";
5670     }
5671
5672     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5673         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5674
5675     my $migration_ip;
5676     my $get_migration_ip = sub {
5677         my ($nodename) = @_;
5678
5679         return $migration_ip if defined($migration_ip);
5680
5681         my $cidr = $migrate_opts->{network};
5682
5683         if (!defined($cidr)) {
5684             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5685             $cidr = $dc_conf->{migration}->{network};
5686         }
5687
5688         if (defined($cidr)) {
5689             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5690
5691             die "could not get IP: no address configured on local " .
5692                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5693
5694             die "could not get IP: multiple addresses configured on local " .
5695                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5696
5697             $migration_ip = @$ips[0];
5698         }
5699
5700         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5701             if !defined($migration_ip);
5702
5703         return $migration_ip;
5704     };
5705
5706     if ($statefile) {
5707         if ($statefile eq 'tcp') {
5708             my $migrate = $res->{migrate} = { proto => 'tcp' };
5709             $migrate->{addr} = "localhost";
5710             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5711             my $nodename = nodename();
5712
5713             if (!defined($migration_type)) {
5714                 if (defined($datacenterconf->{migration}->{type})) {
5715                     $migration_type = $datacenterconf->{migration}->{type};
5716                 } else {
5717                     $migration_type = 'secure';
5718                 }
5719             }
5720
5721             if ($migration_type eq 'insecure') {
5722                 $migrate->{addr} = $get_migration_ip->($nodename);
5723                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5724             }
5725
5726             # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5727             # via QMP later
5728             push @$cmd, '-incoming', 'defer';
5729             push @$cmd, '-S';
5730
5731         } elsif ($statefile eq 'unix') {
5732             # should be default for secure migrations as a ssh TCP forward
5733             # tunnel is not deterministic reliable ready and fails regurarly
5734             # to set up in time, so use UNIX socket forwards
5735             my $migrate = $res->{migrate} = { proto => 'unix' };
5736             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5737             unlink $migrate->{addr};
5738
5739             $migrate->{uri} = "unix:$migrate->{addr}";
5740             push @$cmd, '-incoming', $migrate->{uri};
5741             push @$cmd, '-S';
5742
5743         } elsif (-e $statefile) {
5744             push @$cmd, '-loadstate', $statefile;
5745         } else {
5746             my $statepath = PVE::Storage::path($storecfg, $statefile);
5747             push @$vollist, $statefile;
5748             push @$cmd, '-loadstate', $statepath;
5749         }
5750     } elsif ($params->{paused}) {
5751         push @$cmd, '-S';
5752     }
5753
5754     my $memory = get_current_memory($conf->{memory});
5755     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5756
5757     my $pci_reserve_list = [];
5758     for my $device (values $pci_devices->%*) {
5759         next if $device->{mdev}; # we don't reserve for mdev devices
5760         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5761     }
5762
5763     # reserve all PCI IDs before actually doing anything with them
5764     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5765
5766     eval {
5767         my $uuid;
5768         for my $id (sort keys %$pci_devices) {
5769             my $d = $pci_devices->{$id};
5770             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5771
5772             my $chosen_mdev;
5773             for my $dev ($d->{ids}->@*) {
5774                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5775                 if ($d->{mdev}) {
5776                     warn $@ if $@;
5777                     $chosen_mdev = $info;
5778                     last if $chosen_mdev; # if successful, we're done
5779                 } else {
5780                     die $@ if $@;
5781                 }
5782             }
5783
5784             next if !$d->{mdev};
5785             die "could not create mediated device\n" if !defined($chosen_mdev);
5786
5787             # nvidia grid needs the uuid of the mdev as qemu parameter
5788             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5789                 if (defined($conf->{smbios1})) {
5790                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5791                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5792                 }
5793                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5794             }
5795         }
5796         push @$cmd, '-uuid', $uuid if defined($uuid);
5797     };
5798     if (my $err = $@) {
5799         eval { cleanup_pci_devices($vmid, $conf) };
5800         warn $@ if $@;
5801         die $err;
5802     }
5803
5804     PVE::Storage::activate_volumes($storecfg, $vollist);
5805
5806
5807     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5808     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5809     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5810     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5811     # timeout should be more than enough here...
5812     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5813
5814     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5815
5816     my %run_params = (
5817         timeout => $statefile ? undef : $start_timeout,
5818         umask => 0077,
5819         noerr => 1,
5820     );
5821
5822     # when migrating, prefix QEMU output so other side can pick up any
5823     # errors that might occur and show the user
5824     if ($migratedfrom) {
5825         $run_params{quiet} = 1;
5826         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5827     }
5828
5829     my %systemd_properties = (
5830         Slice => 'qemu.slice',
5831         KillMode => 'process',
5832         SendSIGKILL => 0,
5833         TimeoutStopUSec => ULONG_MAX, # infinity
5834     );
5835
5836     if (PVE::CGroup::cgroup_mode() == 2) {
5837         $systemd_properties{CPUWeight} = $cpuunits;
5838     } else {
5839         $systemd_properties{CPUShares} = $cpuunits;
5840     }
5841
5842     if (my $cpulimit = $conf->{cpulimit}) {
5843         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5844     }
5845     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5846
5847     my $run_qemu = sub {
5848         PVE::Tools::run_fork sub {
5849             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5850
5851             my $tpmpid;
5852             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5853                 # start the TPM emulator so QEMU can connect on start
5854                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5855             }
5856
5857             my $exitcode = run_command($cmd, %run_params);
5858             if ($exitcode) {
5859                 if ($tpmpid) {
5860                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5861                     kill 'TERM', $tpmpid;
5862                 }
5863                 die "QEMU exited with code $exitcode\n";
5864             }
5865         };
5866     };
5867
5868     if ($conf->{hugepages}) {
5869
5870         my $code = sub {
5871             my $hotplug_features =
5872                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5873             my $hugepages_topology =
5874                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5875
5876             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5877
5878             PVE::QemuServer::Memory::hugepages_mount();
5879             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5880
5881             eval { $run_qemu->() };
5882             if (my $err = $@) {
5883                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5884                     if !$conf->{keephugepages};
5885                 die $err;
5886             }
5887
5888             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5889                 if !$conf->{keephugepages};
5890         };
5891         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5892
5893     } else {
5894         eval { $run_qemu->() };
5895     }
5896
5897     if (my $err = $@) {
5898         # deactivate volumes if start fails
5899         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5900         warn $@ if $@;
5901         eval { cleanup_pci_devices($vmid, $conf) };
5902         warn $@ if $@;
5903
5904         die "start failed: $err";
5905     }
5906
5907     # re-reserve all PCI IDs now that we can know the actual VM PID
5908     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5909     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5910     warn $@ if $@;
5911
5912     if (defined(my $migrate = $res->{migrate})) {
5913         if ($migrate->{proto} eq 'tcp') {
5914             my $nodename = nodename();
5915             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5916             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5917             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5918             mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
5919         }
5920         print "migration listens on $migrate->{uri}\n";
5921     } elsif ($statefile) {
5922         eval { mon_cmd($vmid, "cont"); };
5923         warn $@ if $@;
5924     }
5925
5926     #start nbd server for storage migration
5927     if (my $nbd = $migrate_opts->{nbd}) {
5928         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5929
5930         my $migrate_storage_uri;
5931         # nbd_protocol_version > 0 for unix socket support
5932         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5933             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5934             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5935             $migrate_storage_uri = "nbd:unix:$socket_path";
5936             $res->{migrate}->{unix_sockets} = [$socket_path];
5937         } else {
5938             my $nodename = nodename();
5939             my $localip = $get_migration_ip->($nodename);
5940             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5941             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5942
5943             mon_cmd($vmid, "nbd-server-start", addr => {
5944                 type => 'inet',
5945                 data => {
5946                     host => "${localip}",
5947                     port => "${storage_migrate_port}",
5948                 },
5949             });
5950             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5951             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5952         }
5953
5954         my $block_info = mon_cmd($vmid, "query-block");
5955         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5956
5957         foreach my $opt (sort keys %$nbd) {
5958             my $drivestr = $nbd->{$opt}->{drivestr};
5959             my $volid = $nbd->{$opt}->{volid};
5960
5961             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5962
5963             mon_cmd(
5964                 $vmid,
5965                 "block-export-add",
5966                 id => "drive-$opt",
5967                 'node-name' => $block_node,
5968                 writable => JSON::true,
5969                 type => "nbd",
5970                 name => "drive-$opt", # NBD export name
5971             );
5972
5973             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5974             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5975             print "re-using replicated volume: $opt - $volid\n"
5976                 if $nbd->{$opt}->{replicated};
5977
5978             $res->{drives}->{$opt} = $nbd->{$opt};
5979             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5980         }
5981     }
5982
5983     if ($migratedfrom) {
5984         eval {
5985             set_migration_caps($vmid);
5986         };
5987         warn $@ if $@;
5988
5989         if ($spice_port) {
5990             print "spice listens on port $spice_port\n";
5991             $res->{spice_port} = $spice_port;
5992             if ($migrate_opts->{spice_ticket}) {
5993                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5994                     $migrate_opts->{spice_ticket});
5995                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5996             }
5997         }
5998
5999     } else {
6000         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6001             if !$statefile && $conf->{balloon};
6002
6003         foreach my $opt (keys %$conf) {
6004             next if $opt !~  m/^net\d+$/;
6005             my $nicconf = parse_net($conf->{$opt});
6006             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6007         }
6008         add_nets_bridge_fdb($conf, $vmid);
6009     }
6010
6011    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6012         eval {
6013             mon_cmd(
6014                 $vmid,
6015                 'qom-set',
6016                 path => "machine/peripheral/balloon0",
6017                 property => "guest-stats-polling-interval",
6018                 value => 2
6019             );
6020         };
6021         log_warn("could not set polling interval for ballooning - $@") if $@;
6022     }
6023
6024     if ($resume) {
6025         print "Resumed VM, removing state\n";
6026         if (my $vmstate = $conf->{vmstate}) {
6027             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6028             PVE::Storage::vdisk_free($storecfg, $vmstate);
6029         }
6030         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6031         PVE::QemuConfig->write_config($vmid, $conf);
6032     }
6033
6034     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6035
6036     my ($current_machine, $is_deprecated) =
6037         PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6038     if ($is_deprecated) {
6039         log_warn(
6040             "current machine version '$current_machine' is deprecated - see the documentation and ".
6041             "change to a newer one",
6042         );
6043     }
6044
6045     return $res;
6046 }
6047
6048 sub vm_commandline {
6049     my ($storecfg, $vmid, $snapname) = @_;
6050
6051     my $conf = PVE::QemuConfig->load_config($vmid);
6052
6053     my ($forcemachine, $forcecpu);
6054     if ($snapname) {
6055         my $snapshot = $conf->{snapshots}->{$snapname};
6056         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6057
6058         # check for machine or CPU overrides in snapshot
6059         $forcemachine = $snapshot->{runningmachine};
6060         $forcecpu = $snapshot->{runningcpu};
6061
6062         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6063
6064         $conf = $snapshot;
6065     }
6066
6067     my $defaults = load_defaults();
6068
6069     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6070
6071     return PVE::Tools::cmd2string($cmd);
6072 }
6073
6074 sub vm_reset {
6075     my ($vmid, $skiplock) = @_;
6076
6077     PVE::QemuConfig->lock_config($vmid, sub {
6078
6079         my $conf = PVE::QemuConfig->load_config($vmid);
6080
6081         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6082
6083         mon_cmd($vmid, "system_reset");
6084     });
6085 }
6086
6087 sub get_vm_volumes {
6088     my ($conf) = @_;
6089
6090     my $vollist = [];
6091     foreach_volid($conf, sub {
6092         my ($volid, $attr) = @_;
6093
6094         return if $volid =~ m|^/|;
6095
6096         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6097         return if !$sid;
6098
6099         push @$vollist, $volid;
6100     });
6101
6102     return $vollist;
6103 }
6104
6105 sub cleanup_pci_devices {
6106     my ($vmid, $conf) = @_;
6107
6108     foreach my $key (keys %$conf) {
6109         next if $key !~ m/^hostpci(\d+)$/;
6110         my $hostpciindex = $1;
6111         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6112         my $d = parse_hostpci($conf->{$key});
6113         if ($d->{mdev}) {
6114             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6115             # don't want to break ABI just for this two liner
6116             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6117
6118             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6119             # out when we do it first. so wait for 10 seconds and then try it
6120             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/) {
6121                 sleep 10;
6122             }
6123
6124             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6125         }
6126     }
6127     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6128 }
6129
6130 sub vm_stop_cleanup {
6131     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6132
6133     eval {
6134
6135         if (!$keepActive) {
6136             my $vollist = get_vm_volumes($conf);
6137             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6138
6139             if (my $tpmdrive = $conf->{tpmstate0}) {
6140                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6141                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6142                 if ($storeid) {
6143                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6144                 }
6145             }
6146         }
6147
6148         foreach my $ext (qw(mon qmp pid vnc qga)) {
6149             unlink "/var/run/qemu-server/${vmid}.$ext";
6150         }
6151
6152         if ($conf->{ivshmem}) {
6153             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6154             # just delete it for now, VMs which have this already open do not
6155             # are affected, but new VMs will get a separated one. If this
6156             # becomes an issue we either add some sort of ref-counting or just
6157             # add a "don't delete on stop" flag to the ivshmem format.
6158             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6159         }
6160
6161         cleanup_pci_devices($vmid, $conf);
6162
6163         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6164     };
6165     warn $@ if $@; # avoid errors - just warn
6166 }
6167
6168 # call only in locked context
6169 sub _do_vm_stop {
6170     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6171
6172     my $pid = check_running($vmid, $nocheck);
6173     return if !$pid;
6174
6175     my $conf;
6176     if (!$nocheck) {
6177         $conf = PVE::QemuConfig->load_config($vmid);
6178         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6179         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6180             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6181             $timeout = $opts->{down} if $opts->{down};
6182         }
6183         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6184     }
6185
6186     eval {
6187         if ($shutdown) {
6188             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6189                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6190             } else {
6191                 mon_cmd($vmid, "system_powerdown");
6192             }
6193         } else {
6194             mon_cmd($vmid, "quit");
6195         }
6196     };
6197     my $err = $@;
6198
6199     if (!$err) {
6200         $timeout = 60 if !defined($timeout);
6201
6202         my $count = 0;
6203         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6204             $count++;
6205             sleep 1;
6206         }
6207
6208         if ($count >= $timeout) {
6209             if ($force) {
6210                 warn "VM still running - terminating now with SIGTERM\n";
6211                 kill 15, $pid;
6212             } else {
6213                 die "VM quit/powerdown failed - got timeout\n";
6214             }
6215         } else {
6216             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6217             return;
6218         }
6219     } else {
6220        if (!check_running($vmid, $nocheck)) {
6221             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6222             return;
6223        }
6224        if ($force) {
6225             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6226             kill 15, $pid;
6227         } else {
6228             die "VM quit/powerdown failed\n";
6229         }
6230     }
6231
6232     # wait again
6233     $timeout = 10;
6234
6235     my $count = 0;
6236     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6237         $count++;
6238         sleep 1;
6239     }
6240
6241     if ($count >= $timeout) {
6242         warn "VM still running - terminating now with SIGKILL\n";
6243         kill 9, $pid;
6244         sleep 1;
6245     }
6246
6247     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6248 }
6249
6250 # Note: use $nocheck to skip tests if VM configuration file exists.
6251 # We need that when migration VMs to other nodes (files already moved)
6252 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6253 sub vm_stop {
6254     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6255
6256     $force = 1 if !defined($force) && !$shutdown;
6257
6258     if ($migratedfrom){
6259         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6260         kill 15, $pid if $pid;
6261         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6262         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6263         return;
6264     }
6265
6266     PVE::QemuConfig->lock_config($vmid, sub {
6267         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6268    });
6269 }
6270
6271 sub vm_reboot {
6272     my ($vmid, $timeout) = @_;
6273
6274     PVE::QemuConfig->lock_config($vmid, sub {
6275         eval {
6276
6277             # only reboot if running, as qmeventd starts it again on a stop event
6278             return if !check_running($vmid);
6279
6280             create_reboot_request($vmid);
6281
6282             my $storecfg = PVE::Storage::config();
6283             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6284
6285         };
6286         if (my $err = $@) {
6287             # avoid that the next normal shutdown will be confused for a reboot
6288             clear_reboot_request($vmid);
6289             die $err;
6290         }
6291    });
6292 }
6293
6294 # note: if using the statestorage parameter, the caller has to check privileges
6295 sub vm_suspend {
6296     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6297
6298     my $conf;
6299     my $path;
6300     my $storecfg;
6301     my $vmstate;
6302
6303     PVE::QemuConfig->lock_config($vmid, sub {
6304
6305         $conf = PVE::QemuConfig->load_config($vmid);
6306
6307         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6308         PVE::QemuConfig->check_lock($conf)
6309             if !($skiplock || $is_backing_up);
6310
6311         die "cannot suspend to disk during backup\n"
6312             if $is_backing_up && $includestate;
6313
6314         if ($includestate) {
6315             $conf->{lock} = 'suspending';
6316             my $date = strftime("%Y-%m-%d", localtime(time()));
6317             $storecfg = PVE::Storage::config();
6318             if (!$statestorage) {
6319                 $statestorage = find_vmstate_storage($conf, $storecfg);
6320                 # check permissions for the storage
6321                 my $rpcenv = PVE::RPCEnvironment::get();
6322                 if ($rpcenv->{type} ne 'cli') {
6323                     my $authuser = $rpcenv->get_user();
6324                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6325                 }
6326             }
6327
6328
6329             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6330                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6331             $path = PVE::Storage::path($storecfg, $vmstate);
6332             PVE::QemuConfig->write_config($vmid, $conf);
6333         } else {
6334             mon_cmd($vmid, "stop");
6335         }
6336     });
6337
6338     if ($includestate) {
6339         # save vm state
6340         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6341
6342         eval {
6343             set_migration_caps($vmid, 1);
6344             mon_cmd($vmid, "savevm-start", statefile => $path);
6345             for(;;) {
6346                 my $state = mon_cmd($vmid, "query-savevm");
6347                 if (!$state->{status}) {
6348                     die "savevm not active\n";
6349                 } elsif ($state->{status} eq 'active') {
6350                     sleep(1);
6351                     next;
6352                 } elsif ($state->{status} eq 'completed') {
6353                     print "State saved, quitting\n";
6354                     last;
6355                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6356                     die "query-savevm failed with error '$state->{error}'\n"
6357                 } else {
6358                     die "query-savevm returned status '$state->{status}'\n";
6359                 }
6360             }
6361         };
6362         my $err = $@;
6363
6364         PVE::QemuConfig->lock_config($vmid, sub {
6365             $conf = PVE::QemuConfig->load_config($vmid);
6366             if ($err) {
6367                 # cleanup, but leave suspending lock, to indicate something went wrong
6368                 eval {
6369                     mon_cmd($vmid, "savevm-end");
6370                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6371                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6372                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6373                     PVE::QemuConfig->write_config($vmid, $conf);
6374                 };
6375                 warn $@ if $@;
6376                 die $err;
6377             }
6378
6379             die "lock changed unexpectedly\n"
6380                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6381
6382             mon_cmd($vmid, "quit");
6383             $conf->{lock} = 'suspended';
6384             PVE::QemuConfig->write_config($vmid, $conf);
6385         });
6386     }
6387 }
6388
6389 # $nocheck is set when called as part of a migration - in this context the
6390 # location of the config file (source or target node) is not deterministic,
6391 # since migration cannot wait for pmxcfs to process the rename
6392 sub vm_resume {
6393     my ($vmid, $skiplock, $nocheck) = @_;
6394
6395     PVE::QemuConfig->lock_config($vmid, sub {
6396         my $res = mon_cmd($vmid, 'query-status');
6397         my $resume_cmd = 'cont';
6398         my $reset = 0;
6399         my $conf;
6400         if ($nocheck) {
6401             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6402             if ($@) {
6403                 my $vmlist = PVE::Cluster::get_vmlist();
6404                 if (exists($vmlist->{ids}->{$vmid})) {
6405                     my $node = $vmlist->{ids}->{$vmid}->{node};
6406                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6407                 }
6408                 if (!$conf) {
6409                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6410                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6411                 }
6412             }
6413         } else {
6414             $conf = PVE::QemuConfig->load_config($vmid);
6415         }
6416
6417         if ($res->{status}) {
6418             return if $res->{status} eq 'running'; # job done, go home
6419             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6420             $reset = 1 if $res->{status} eq 'shutdown';
6421         }
6422
6423         if (!$nocheck) {
6424             PVE::QemuConfig->check_lock($conf)
6425                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6426         }
6427
6428         if ($reset) {
6429             # required if a VM shuts down during a backup and we get a resume
6430             # request before the backup finishes for example
6431             mon_cmd($vmid, "system_reset");
6432         }
6433
6434         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6435
6436         mon_cmd($vmid, $resume_cmd);
6437     });
6438 }
6439
6440 sub vm_sendkey {
6441     my ($vmid, $skiplock, $key) = @_;
6442
6443     PVE::QemuConfig->lock_config($vmid, sub {
6444
6445         my $conf = PVE::QemuConfig->load_config($vmid);
6446
6447         # there is no qmp command, so we use the human monitor command
6448         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6449         die $res if $res ne '';
6450     });
6451 }
6452
6453 sub check_bridge_access {
6454     my ($rpcenv, $authuser, $conf) = @_;
6455
6456     return 1 if $authuser eq 'root@pam';
6457
6458     for my $opt (sort keys $conf->%*) {
6459         next if $opt !~ m/^net\d+$/;
6460         my $net = parse_net($conf->{$opt});
6461         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6462         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6463     }
6464     return 1;
6465 };
6466
6467 sub check_mapping_access {
6468     my ($rpcenv, $user, $conf) = @_;
6469
6470     for my $opt (keys $conf->%*) {
6471         if ($opt =~ m/^usb\d+$/) {
6472             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6473             if (my $host = $device->{host}) {
6474                 die "only root can set '$opt' config for real devices\n"
6475                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6476             } elsif ($device->{mapping}) {
6477                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6478             } else {
6479                 die "either 'host' or 'mapping' must be set.\n";
6480             }
6481         } elsif ($opt =~ m/^hostpci\d+$/) {
6482             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6483             if ($device->{host}) {
6484                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6485             } elsif ($device->{mapping}) {
6486                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6487             } else {
6488                 die "either 'host' or 'mapping' must be set.\n";
6489             }
6490        }
6491    }
6492 };
6493
6494 sub check_restore_permissions {
6495     my ($rpcenv, $user, $conf) = @_;
6496
6497     check_bridge_access($rpcenv, $user, $conf);
6498     check_mapping_access($rpcenv, $user, $conf);
6499 }
6500 # vzdump restore implementaion
6501
6502 sub tar_archive_read_firstfile {
6503     my $archive = shift;
6504
6505     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6506
6507     # try to detect archive type first
6508     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6509         die "unable to open file '$archive'\n";
6510     my $firstfile = <$fh>;
6511     kill 15, $pid;
6512     close $fh;
6513
6514     die "ERROR: archive contaions no data\n" if !$firstfile;
6515     chomp $firstfile;
6516
6517     return $firstfile;
6518 }
6519
6520 sub tar_restore_cleanup {
6521     my ($storecfg, $statfile) = @_;
6522
6523     print STDERR "starting cleanup\n";
6524
6525     if (my $fd = IO::File->new($statfile, "r")) {
6526         while (defined(my $line = <$fd>)) {
6527             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6528                 my $volid = $2;
6529                 eval {
6530                     if ($volid =~ m|^/|) {
6531                         unlink $volid || die 'unlink failed\n';
6532                     } else {
6533                         PVE::Storage::vdisk_free($storecfg, $volid);
6534                     }
6535                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6536                 };
6537                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6538             } else {
6539                 print STDERR "unable to parse line in statfile - $line";
6540             }
6541         }
6542         $fd->close();
6543     }
6544 }
6545
6546 sub restore_file_archive {
6547     my ($archive, $vmid, $user, $opts) = @_;
6548
6549     return restore_vma_archive($archive, $vmid, $user, $opts)
6550         if $archive eq '-';
6551
6552     my $info = PVE::Storage::archive_info($archive);
6553     my $format = $opts->{format} // $info->{format};
6554     my $comp = $info->{compression};
6555
6556     # try to detect archive format
6557     if ($format eq 'tar') {
6558         return restore_tar_archive($archive, $vmid, $user, $opts);
6559     } else {
6560         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6561     }
6562 }
6563
6564 # hepler to remove disks that will not be used after restore
6565 my $restore_cleanup_oldconf = sub {
6566     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6567
6568     my $kept_disks = {};
6569
6570     PVE::QemuConfig->foreach_volume($oldconf, sub {
6571         my ($ds, $drive) = @_;
6572
6573         return if drive_is_cdrom($drive, 1);
6574
6575         my $volid = $drive->{file};
6576         return if !$volid || $volid =~ m|^/|;
6577
6578         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6579         return if !$path || !$owner || ($owner != $vmid);
6580
6581         # Note: only delete disk we want to restore
6582         # other volumes will become unused
6583         if ($virtdev_hash->{$ds}) {
6584             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6585             if (my $err = $@) {
6586                 warn $err;
6587             }
6588         } else {
6589             $kept_disks->{$volid} = 1;
6590         }
6591     });
6592
6593     # after the restore we have no snapshots anymore
6594     for my $snapname (keys $oldconf->{snapshots}->%*) {
6595         my $snap = $oldconf->{snapshots}->{$snapname};
6596         if ($snap->{vmstate}) {
6597             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6598             if (my $err = $@) {
6599                 warn $err;
6600             }
6601         }
6602
6603         for my $volid (keys $kept_disks->%*) {
6604             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6605             warn $@ if $@;
6606         }
6607     }
6608 };
6609
6610 # Helper to parse vzdump backup device hints
6611 #
6612 # $rpcenv: Environment, used to ckeck storage permissions
6613 # $user: User ID, to check storage permissions
6614 # $storecfg: Storage configuration
6615 # $fh: the file handle for reading the configuration
6616 # $devinfo: should contain device sizes for all backu-up'ed devices
6617 # $options: backup options (pool, default storage)
6618 #
6619 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6620 my $parse_backup_hints = sub {
6621     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6622
6623     my $check_storage = sub { # assert if an image can be allocate
6624         my ($storeid, $scfg) = @_;
6625         die "Content type 'images' is not available on storage '$storeid'\n"
6626             if !$scfg->{content}->{images};
6627         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6628             if $user ne 'root@pam';
6629     };
6630
6631     my $virtdev_hash = {};
6632     while (defined(my $line = <$fh>)) {
6633         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6634             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6635             die "archive does not contain data for drive '$virtdev'\n"
6636                 if !$devinfo->{$devname};
6637
6638             if (defined($options->{storage})) {
6639                 $storeid = $options->{storage} || 'local';
6640             } elsif (!$storeid) {
6641                 $storeid = 'local';
6642             }
6643             $format = 'raw' if !$format;
6644             $devinfo->{$devname}->{devname} = $devname;
6645             $devinfo->{$devname}->{virtdev} = $virtdev;
6646             $devinfo->{$devname}->{format} = $format;
6647             $devinfo->{$devname}->{storeid} = $storeid;
6648
6649             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6650             $check_storage->($storeid, $scfg); # permission and content type check
6651
6652             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6653         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6654             my $virtdev = $1;
6655             my $drive = parse_drive($virtdev, $2);
6656
6657             if (drive_is_cloudinit($drive)) {
6658                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6659                 $storeid = $options->{storage} if defined ($options->{storage});
6660                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6661                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6662
6663                 $check_storage->($storeid, $scfg); # permission and content type check
6664
6665                 $virtdev_hash->{$virtdev} = {
6666                     format => $format,
6667                     storeid => $storeid,
6668                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6669                     is_cloudinit => 1,
6670                 };
6671             }
6672         }
6673     }
6674
6675     return $virtdev_hash;
6676 };
6677
6678 # Helper to allocate and activate all volumes required for a restore
6679 #
6680 # $storecfg: Storage configuration
6681 # $virtdev_hash: as returned by parse_backup_hints()
6682 #
6683 # Returns: { $virtdev => $volid }
6684 my $restore_allocate_devices = sub {
6685     my ($storecfg, $virtdev_hash, $vmid) = @_;
6686
6687     my $map = {};
6688     foreach my $virtdev (sort keys %$virtdev_hash) {
6689         my $d = $virtdev_hash->{$virtdev};
6690         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6691         my $storeid = $d->{storeid};
6692         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6693
6694         # test if requested format is supported
6695         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6696         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6697         $d->{format} = $defFormat if !$supported;
6698
6699         my $name;
6700         if ($d->{is_cloudinit}) {
6701             $name = "vm-$vmid-cloudinit";
6702             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6703             if ($scfg->{path}) {
6704                 $name .= ".$d->{format}";
6705             }
6706         }
6707
6708         my $volid = PVE::Storage::vdisk_alloc(
6709             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6710
6711         print STDERR "new volume ID is '$volid'\n";
6712         $d->{volid} = $volid;
6713
6714         PVE::Storage::activate_volumes($storecfg, [$volid]);
6715
6716         $map->{$virtdev} = $volid;
6717     }
6718
6719     return $map;
6720 };
6721
6722 sub restore_update_config_line {
6723     my ($cookie, $map, $line, $unique) = @_;
6724
6725     return '' if $line =~ m/^\#qmdump\#/;
6726     return '' if $line =~ m/^\#vzdump\#/;
6727     return '' if $line =~ m/^lock:/;
6728     return '' if $line =~ m/^unused\d+:/;
6729     return '' if $line =~ m/^parent:/;
6730
6731     my $res = '';
6732
6733     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6734     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6735         # try to convert old 1.X settings
6736         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6737         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6738             my ($model, $macaddr) = split(/\=/, $devconfig);
6739             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6740             my $net = {
6741                 model => $model,
6742                 bridge => "vmbr$ind",
6743                 macaddr => $macaddr,
6744             };
6745             my $netstr = print_net($net);
6746
6747             $res .= "net$cookie->{netcount}: $netstr\n";
6748             $cookie->{netcount}++;
6749         }
6750     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6751         my ($id, $netstr) = ($1, $2);
6752         my $net = parse_net($netstr);
6753         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6754         $netstr = print_net($net);
6755         $res .= "$id: $netstr\n";
6756     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6757         my $virtdev = $1;
6758         my $value = $3;
6759         my $di = parse_drive($virtdev, $value);
6760         if (defined($di->{backup}) && !$di->{backup}) {
6761             $res .= "#$line";
6762         } elsif ($map->{$virtdev}) {
6763             delete $di->{format}; # format can change on restore
6764             $di->{file} = $map->{$virtdev};
6765             $value = print_drive($di);
6766             $res .= "$virtdev: $value\n";
6767         } else {
6768             $res .= $line;
6769         }
6770     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6771         my $vmgenid = $1;
6772         if ($vmgenid ne '0') {
6773             # always generate a new vmgenid if there was a valid one setup
6774             $vmgenid = generate_uuid();
6775         }
6776         $res .= "vmgenid: $vmgenid\n";
6777     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6778         my ($uuid, $uuid_str);
6779         UUID::generate($uuid);
6780         UUID::unparse($uuid, $uuid_str);
6781         my $smbios1 = parse_smbios1($2);
6782         $smbios1->{uuid} = $uuid_str;
6783         $res .= $1.print_smbios1($smbios1)."\n";
6784     } else {
6785         $res .= $line;
6786     }
6787
6788     return $res;
6789 }
6790
6791 my $restore_deactivate_volumes = sub {
6792     my ($storecfg, $virtdev_hash) = @_;
6793
6794     my $vollist = [];
6795     for my $dev (values $virtdev_hash->%*) {
6796         push $vollist->@*, $dev->{volid} if $dev->{volid};
6797     }
6798
6799     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6800     print STDERR $@ if $@;
6801 };
6802
6803 my $restore_destroy_volumes = sub {
6804     my ($storecfg, $virtdev_hash) = @_;
6805
6806     for my $dev (values $virtdev_hash->%*) {
6807         my $volid = $dev->{volid} or next;
6808         eval {
6809             PVE::Storage::vdisk_free($storecfg, $volid);
6810             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6811         };
6812         print STDERR "unable to cleanup '$volid' - $@" if $@;
6813     }
6814 };
6815
6816 sub restore_merge_config {
6817     my ($filename, $backup_conf_raw, $override_conf) = @_;
6818
6819     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6820     for my $key (keys $override_conf->%*) {
6821         $backup_conf->{$key} = $override_conf->{$key};
6822     }
6823
6824     return $backup_conf;
6825 }
6826
6827 sub scan_volids {
6828     my ($cfg, $vmid) = @_;
6829
6830     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6831
6832     my $volid_hash = {};
6833     foreach my $storeid (keys %$info) {
6834         foreach my $item (@{$info->{$storeid}}) {
6835             next if !($item->{volid} && $item->{size});
6836             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6837             $volid_hash->{$item->{volid}} = $item;
6838         }
6839     }
6840
6841     return $volid_hash;
6842 }
6843
6844 sub update_disk_config {
6845     my ($vmid, $conf, $volid_hash) = @_;
6846
6847     my $changes;
6848     my $prefix = "VM $vmid";
6849
6850     # used and unused disks
6851     my $referenced = {};
6852
6853     # Note: it is allowed to define multiple storages with same path (alias), so
6854     # we need to check both 'volid' and real 'path' (two different volid can point
6855     # to the same path).
6856
6857     my $referencedpath = {};
6858
6859     # update size info
6860     PVE::QemuConfig->foreach_volume($conf, sub {
6861         my ($opt, $drive) = @_;
6862
6863         my $volid = $drive->{file};
6864         return if !$volid;
6865         my $volume = $volid_hash->{$volid};
6866
6867         # mark volid as "in-use" for next step
6868         $referenced->{$volid} = 1;
6869         if ($volume && (my $path = $volume->{path})) {
6870             $referencedpath->{$path} = 1;
6871         }
6872
6873         return if drive_is_cdrom($drive);
6874         return if !$volume;
6875
6876         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6877         if (defined($updated)) {
6878             $changes = 1;
6879             $conf->{$opt} = print_drive($updated);
6880             print "$prefix ($opt): $msg\n";
6881         }
6882     });
6883
6884     # remove 'unusedX' entry if volume is used
6885     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6886         my ($opt, $drive) = @_;
6887
6888         my $volid = $drive->{file};
6889         return if !$volid;
6890
6891         my $path;
6892         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6893         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6894             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6895             $changes = 1;
6896             delete $conf->{$opt};
6897         }
6898
6899         $referenced->{$volid} = 1;
6900         $referencedpath->{$path} = 1 if $path;
6901     });
6902
6903     foreach my $volid (sort keys %$volid_hash) {
6904         next if $volid =~ m/vm-$vmid-state-/;
6905         next if $referenced->{$volid};
6906         my $path = $volid_hash->{$volid}->{path};
6907         next if !$path; # just to be sure
6908         next if $referencedpath->{$path};
6909         $changes = 1;
6910         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6911         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6912         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6913     }
6914
6915     return $changes;
6916 }
6917
6918 sub rescan {
6919     my ($vmid, $nolock, $dryrun) = @_;
6920
6921     my $cfg = PVE::Storage::config();
6922
6923     print "rescan volumes...\n";
6924     my $volid_hash = scan_volids($cfg, $vmid);
6925
6926     my $updatefn =  sub {
6927         my ($vmid) = @_;
6928
6929         my $conf = PVE::QemuConfig->load_config($vmid);
6930
6931         PVE::QemuConfig->check_lock($conf);
6932
6933         my $vm_volids = {};
6934         foreach my $volid (keys %$volid_hash) {
6935             my $info = $volid_hash->{$volid};
6936             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6937         }
6938
6939         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6940
6941         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6942     };
6943
6944     if (defined($vmid)) {
6945         if ($nolock) {
6946             &$updatefn($vmid);
6947         } else {
6948             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6949         }
6950     } else {
6951         my $vmlist = config_list();
6952         foreach my $vmid (keys %$vmlist) {
6953             if ($nolock) {
6954                 &$updatefn($vmid);
6955             } else {
6956                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6957             }
6958         }
6959     }
6960 }
6961
6962 sub restore_proxmox_backup_archive {
6963     my ($archive, $vmid, $user, $options) = @_;
6964
6965     my $storecfg = PVE::Storage::config();
6966
6967     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6968     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6969
6970     my $fingerprint = $scfg->{fingerprint};
6971     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6972
6973     my $repo = PVE::PBSClient::get_repository($scfg);
6974     my $namespace = $scfg->{namespace};
6975
6976     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6977     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6978     local $ENV{PBS_PASSWORD} = $password;
6979     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6980
6981     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6982         PVE::Storage::parse_volname($storecfg, $archive);
6983
6984     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6985
6986     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6987
6988     my $tmpdir = "/var/tmp/vzdumptmp$$";
6989     rmtree $tmpdir;
6990     mkpath $tmpdir;
6991
6992     my $conffile = PVE::QemuConfig->config_file($vmid);
6993      # disable interrupts (always do cleanups)
6994     local $SIG{INT} =
6995         local $SIG{TERM} =
6996         local $SIG{QUIT} =
6997         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6998
6999     # Note: $oldconf is undef if VM does not exists
7000     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7001     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7002     my $new_conf_raw = '';
7003
7004     my $rpcenv = PVE::RPCEnvironment::get();
7005     my $devinfo = {}; # info about drives included in backup
7006     my $virtdev_hash = {}; # info about allocated drives
7007
7008     eval {
7009         # enable interrupts
7010         local $SIG{INT} =
7011             local $SIG{TERM} =
7012             local $SIG{QUIT} =
7013             local $SIG{HUP} =
7014             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7015
7016         my $cfgfn = "$tmpdir/qemu-server.conf";
7017         my $firewall_config_fn = "$tmpdir/fw.conf";
7018         my $index_fn = "$tmpdir/index.json";
7019
7020         my $cmd = "restore";
7021
7022         my $param = [$pbs_backup_name, "index.json", $index_fn];
7023         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7024         my $index = PVE::Tools::file_get_contents($index_fn);
7025         $index = decode_json($index);
7026
7027         foreach my $info (@{$index->{files}}) {
7028             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7029                 my $devname = $1;
7030                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7031                     $devinfo->{$devname}->{size} = $1;
7032                 } else {
7033                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7034                 }
7035             }
7036         }
7037
7038         my $is_qemu_server_backup = scalar(
7039             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7040         );
7041         if (!$is_qemu_server_backup) {
7042             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7043         }
7044         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7045
7046         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7047         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7048
7049         if ($has_firewall_config) {
7050             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7051             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7052
7053             my $pve_firewall_dir = '/etc/pve/firewall';
7054             mkdir $pve_firewall_dir; # make sure the dir exists
7055             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7056         }
7057
7058         my $fh = IO::File->new($cfgfn, "r") ||
7059             die "unable to read qemu-server.conf - $!\n";
7060
7061         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7062
7063         # fixme: rate limit?
7064
7065         # create empty/temp config
7066         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7067
7068         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7069
7070         # allocate volumes
7071         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7072
7073         foreach my $virtdev (sort keys %$virtdev_hash) {
7074             my $d = $virtdev_hash->{$virtdev};
7075             next if $d->{is_cloudinit}; # no need to restore cloudinit
7076
7077             # this fails if storage is unavailable
7078             my $volid = $d->{volid};
7079             my $path = PVE::Storage::path($storecfg, $volid);
7080
7081             # for live-restore we only want to preload the efidisk and TPM state
7082             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7083
7084             my @ns_arg;
7085             if (defined(my $ns = $scfg->{namespace})) {
7086                 @ns_arg = ('--ns', $ns);
7087             }
7088
7089             my $pbs_restore_cmd = [
7090                 '/usr/bin/pbs-restore',
7091                 '--repository', $repo,
7092                 @ns_arg,
7093                 $pbs_backup_name,
7094                 "$d->{devname}.img.fidx",
7095                 $path,
7096                 '--verbose',
7097                 ];
7098
7099             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7100             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7101
7102             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7103                 push @$pbs_restore_cmd, '--skip-zero';
7104             }
7105
7106             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7107             print "restore proxmox backup image: $dbg_cmdstring\n";
7108             run_command($pbs_restore_cmd);
7109         }
7110
7111         $fh->seek(0, 0) || die "seek failed - $!\n";
7112
7113         my $cookie = { netcount => 0 };
7114         while (defined(my $line = <$fh>)) {
7115             $new_conf_raw .= restore_update_config_line(
7116                 $cookie,
7117                 $map,
7118                 $line,
7119                 $options->{unique},
7120             );
7121         }
7122
7123         $fh->close();
7124     };
7125     my $err = $@;
7126
7127     if ($err || !$options->{live}) {
7128         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7129     }
7130
7131     rmtree $tmpdir;
7132
7133     if ($err) {
7134         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7135         die $err;
7136     }
7137
7138     if ($options->{live}) {
7139         # keep lock during live-restore
7140         $new_conf_raw .= "\nlock: create";
7141     }
7142
7143     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7144     check_restore_permissions($rpcenv, $user, $new_conf);
7145     PVE::QemuConfig->write_config($vmid, $new_conf);
7146
7147     eval { rescan($vmid, 1); };
7148     warn $@ if $@;
7149
7150     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7151
7152     if ($options->{live}) {
7153         # enable interrupts
7154         local $SIG{INT} =
7155             local $SIG{TERM} =
7156             local $SIG{QUIT} =
7157             local $SIG{HUP} =
7158             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7159
7160         my $conf = PVE::QemuConfig->load_config($vmid);
7161         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7162
7163         # these special drives are already restored before start
7164         delete $devinfo->{'drive-efidisk0'};
7165         delete $devinfo->{'drive-tpmstate0-backup'};
7166
7167         my $pbs_opts = {
7168             repo => $repo,
7169             keyfile => $keyfile,
7170             snapshot => $pbs_backup_name,
7171             namespace => $namespace,
7172         };
7173         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7174
7175         PVE::QemuConfig->remove_lock($vmid, "create");
7176     }
7177 }
7178
7179 sub pbs_live_restore {
7180     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7181
7182     print "starting VM for live-restore\n";
7183     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7184
7185     my $pbs_backing = {};
7186     for my $ds (keys %$restored_disks) {
7187         $ds =~ m/^drive-(.*)$/;
7188         my $confname = $1;
7189         $pbs_backing->{$confname} = {
7190             repository => $opts->{repo},
7191             snapshot => $opts->{snapshot},
7192             archive => "$ds.img.fidx",
7193         };
7194         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7195         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7196
7197         my $drive = parse_drive($confname, $conf->{$confname});
7198         print "restoring '$ds' to '$drive->{file}'\n";
7199     }
7200
7201     my $drives_streamed = 0;
7202     eval {
7203         # make sure HA doesn't interrupt our restore by stopping the VM
7204         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7205             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7206         }
7207
7208         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7209         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7210         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7211
7212         my $qmeventd_fd = register_qmeventd_handle($vmid);
7213
7214         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7215         # this will effectively collapse the backing image chain consisting of
7216         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7217         # removes itself once all backing images vanish with 'auto-remove=on')
7218         my $jobs = {};
7219         for my $ds (sort keys %$restored_disks) {
7220             my $job_id = "restore-$ds";
7221             mon_cmd($vmid, 'block-stream',
7222                 'job-id' => $job_id,
7223                 device => "$ds",
7224             );
7225             $jobs->{$job_id} = {};
7226         }
7227
7228         mon_cmd($vmid, 'cont');
7229         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7230
7231         print "restore-drive jobs finished successfully, removing all tracking block devices"
7232             ." to disconnect from Proxmox Backup Server\n";
7233
7234         for my $ds (sort keys %$restored_disks) {
7235             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7236         }
7237
7238         close($qmeventd_fd);
7239     };
7240
7241     my $err = $@;
7242
7243     if ($err) {
7244         warn "An error occurred during live-restore: $err\n";
7245         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7246         die "live-restore failed\n";
7247     }
7248 }
7249
7250 sub restore_vma_archive {
7251     my ($archive, $vmid, $user, $opts, $comp) = @_;
7252
7253     my $readfrom = $archive;
7254
7255     my $cfg = PVE::Storage::config();
7256     my $commands = [];
7257     my $bwlimit = $opts->{bwlimit};
7258
7259     my $dbg_cmdstring = '';
7260     my $add_pipe = sub {
7261         my ($cmd) = @_;
7262         push @$commands, $cmd;
7263         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7264         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7265         $readfrom = '-';
7266     };
7267
7268     my $input = undef;
7269     if ($archive eq '-') {
7270         $input = '<&STDIN';
7271     } else {
7272         # If we use a backup from a PVE defined storage we also consider that
7273         # storage's rate limit:
7274         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7275         if (defined($volid)) {
7276             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7277             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7278             if ($readlimit) {
7279                 print STDERR "applying read rate limit: $readlimit\n";
7280                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7281                 $add_pipe->($cstream);
7282             }
7283         }
7284     }
7285
7286     if ($comp) {
7287         my $info = PVE::Storage::decompressor_info('vma', $comp);
7288         my $cmd = $info->{decompressor};
7289         push @$cmd, $readfrom;
7290         $add_pipe->($cmd);
7291     }
7292
7293     my $tmpdir = "/var/tmp/vzdumptmp$$";
7294     rmtree $tmpdir;
7295
7296     # disable interrupts (always do cleanups)
7297     local $SIG{INT} =
7298         local $SIG{TERM} =
7299         local $SIG{QUIT} =
7300         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7301
7302     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7303     POSIX::mkfifo($mapfifo, 0600);
7304     my $fifofh;
7305     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7306
7307     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7308
7309     my $devinfo = {}; # info about drives included in backup
7310     my $virtdev_hash = {}; # info about allocated drives
7311
7312     my $rpcenv = PVE::RPCEnvironment::get();
7313
7314     my $conffile = PVE::QemuConfig->config_file($vmid);
7315
7316     # Note: $oldconf is undef if VM does not exist
7317     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7318     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7319     my $new_conf_raw = '';
7320
7321     my %storage_limits;
7322
7323     my $print_devmap = sub {
7324         my $cfgfn = "$tmpdir/qemu-server.conf";
7325
7326         # we can read the config - that is already extracted
7327         my $fh = IO::File->new($cfgfn, "r") ||
7328             die "unable to read qemu-server.conf - $!\n";
7329
7330         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7331         if (-f $fwcfgfn) {
7332             my $pve_firewall_dir = '/etc/pve/firewall';
7333             mkdir $pve_firewall_dir; # make sure the dir exists
7334             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7335         }
7336
7337         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7338
7339         foreach my $info (values %{$virtdev_hash}) {
7340             my $storeid = $info->{storeid};
7341             next if defined($storage_limits{$storeid});
7342
7343             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7344             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7345             $storage_limits{$storeid} = $limit * 1024;
7346         }
7347
7348         foreach my $devname (keys %$devinfo) {
7349             die "found no device mapping information for device '$devname'\n"
7350                 if !$devinfo->{$devname}->{virtdev};
7351         }
7352
7353         # create empty/temp config
7354         if ($oldconf) {
7355             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7356             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7357         }
7358
7359         # allocate volumes
7360         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7361
7362         # print restore information to $fifofh
7363         foreach my $virtdev (sort keys %$virtdev_hash) {
7364             my $d = $virtdev_hash->{$virtdev};
7365             next if $d->{is_cloudinit}; # no need to restore cloudinit
7366
7367             my $storeid = $d->{storeid};
7368             my $volid = $d->{volid};
7369
7370             my $map_opts = '';
7371             if (my $limit = $storage_limits{$storeid}) {
7372                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7373             }
7374
7375             my $write_zeros = 1;
7376             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7377                 $write_zeros = 0;
7378             }
7379
7380             my $path = PVE::Storage::path($cfg, $volid);
7381
7382             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7383
7384             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7385         }
7386
7387         $fh->seek(0, 0) || die "seek failed - $!\n";
7388
7389         my $cookie = { netcount => 0 };
7390         while (defined(my $line = <$fh>)) {
7391             $new_conf_raw .= restore_update_config_line(
7392                 $cookie,
7393                 $map,
7394                 $line,
7395                 $opts->{unique},
7396             );
7397         }
7398
7399         $fh->close();
7400     };
7401
7402     my $oldtimeout;
7403
7404     eval {
7405         # enable interrupts
7406         local $SIG{INT} =
7407             local $SIG{TERM} =
7408             local $SIG{QUIT} =
7409             local $SIG{HUP} =
7410             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7411         local $SIG{ALRM} = sub { die "got timeout\n"; };
7412
7413         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7414
7415         my $parser = sub {
7416             my $line = shift;
7417
7418             print "$line\n";
7419
7420             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7421                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7422                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7423             } elsif ($line =~ m/^CTIME: /) {
7424                 # we correctly received the vma config, so we can disable
7425                 # the timeout now for disk allocation
7426                 alarm($oldtimeout || 0);
7427                 $oldtimeout = undef;
7428                 &$print_devmap();
7429                 print $fifofh "done\n";
7430                 close($fifofh);
7431                 $fifofh = undef;
7432             }
7433         };
7434
7435         print "restore vma archive: $dbg_cmdstring\n";
7436         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7437     };
7438     my $err = $@;
7439
7440     alarm($oldtimeout) if $oldtimeout;
7441
7442     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7443
7444     close($fifofh) if $fifofh;
7445     unlink $mapfifo;
7446     rmtree $tmpdir;
7447
7448     if ($err) {
7449         $restore_destroy_volumes->($cfg, $virtdev_hash);
7450         die $err;
7451     }
7452
7453     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7454     check_restore_permissions($rpcenv, $user, $new_conf);
7455     PVE::QemuConfig->write_config($vmid, $new_conf);
7456
7457     eval { rescan($vmid, 1); };
7458     warn $@ if $@;
7459
7460     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7461 }
7462
7463 sub restore_tar_archive {
7464     my ($archive, $vmid, $user, $opts) = @_;
7465
7466     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7467         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7468         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7469     }
7470
7471     if ($archive ne '-') {
7472         my $firstfile = tar_archive_read_firstfile($archive);
7473         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7474             if $firstfile ne 'qemu-server.conf';
7475     }
7476
7477     my $storecfg = PVE::Storage::config();
7478
7479     # avoid zombie disks when restoring over an existing VM -> cleanup first
7480     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7481     # skiplock=1 because qmrestore has set the 'create' lock itself already
7482     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7483     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7484
7485     my $tocmd = "/usr/lib/qemu-server/qmextract";
7486
7487     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7488     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7489     $tocmd .= ' --prealloc' if $opts->{prealloc};
7490     $tocmd .= ' --info' if $opts->{info};
7491
7492     # tar option "xf" does not autodetect compression when read from STDIN,
7493     # so we pipe to zcat
7494     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7495         PVE::Tools::shellquote("--to-command=$tocmd");
7496
7497     my $tmpdir = "/var/tmp/vzdumptmp$$";
7498     mkpath $tmpdir;
7499
7500     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7501     local $ENV{VZDUMP_VMID} = $vmid;
7502     local $ENV{VZDUMP_USER} = $user;
7503
7504     my $conffile = PVE::QemuConfig->config_file($vmid);
7505     my $new_conf_raw = '';
7506
7507     # disable interrupts (always do cleanups)
7508     local $SIG{INT} =
7509         local $SIG{TERM} =
7510         local $SIG{QUIT} =
7511         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7512
7513     eval {
7514         # enable interrupts
7515         local $SIG{INT} =
7516             local $SIG{TERM} =
7517             local $SIG{QUIT} =
7518             local $SIG{HUP} =
7519             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7520
7521         if ($archive eq '-') {
7522             print "extracting archive from STDIN\n";
7523             run_command($cmd, input => "<&STDIN");
7524         } else {
7525             print "extracting archive '$archive'\n";
7526             run_command($cmd);
7527         }
7528
7529         return if $opts->{info};
7530
7531         # read new mapping
7532         my $map = {};
7533         my $statfile = "$tmpdir/qmrestore.stat";
7534         if (my $fd = IO::File->new($statfile, "r")) {
7535             while (defined (my $line = <$fd>)) {
7536                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7537                     $map->{$1} = $2 if $1;
7538                 } else {
7539                     print STDERR "unable to parse line in statfile - $line\n";
7540                 }
7541             }
7542             $fd->close();
7543         }
7544
7545         my $confsrc = "$tmpdir/qemu-server.conf";
7546
7547         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7548
7549         my $cookie = { netcount => 0 };
7550         while (defined (my $line = <$srcfd>)) {
7551             $new_conf_raw .= restore_update_config_line(
7552                 $cookie,
7553                 $map,
7554                 $line,
7555                 $opts->{unique},
7556             );
7557         }
7558
7559         $srcfd->close();
7560     };
7561     if (my $err = $@) {
7562         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7563         die $err;
7564     }
7565
7566     rmtree $tmpdir;
7567
7568     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7569
7570     PVE::Cluster::cfs_update(); # make sure we read new file
7571
7572     eval { rescan($vmid, 1); };
7573     warn $@ if $@;
7574 };
7575
7576 sub foreach_storage_used_by_vm {
7577     my ($conf, $func) = @_;
7578
7579     my $sidhash = {};
7580
7581     PVE::QemuConfig->foreach_volume($conf, sub {
7582         my ($ds, $drive) = @_;
7583         return if drive_is_cdrom($drive);
7584
7585         my $volid = $drive->{file};
7586
7587         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7588         $sidhash->{$sid} = $sid if $sid;
7589     });
7590
7591     foreach my $sid (sort keys %$sidhash) {
7592         &$func($sid);
7593     }
7594 }
7595
7596 my $qemu_snap_storage = {
7597     rbd => 1,
7598 };
7599 sub do_snapshots_with_qemu {
7600     my ($storecfg, $volid, $deviceid) = @_;
7601
7602     return if $deviceid =~ m/tpmstate0/;
7603
7604     my $storage_name = PVE::Storage::parse_volume_id($volid);
7605     my $scfg = $storecfg->{ids}->{$storage_name};
7606     die "could not find storage '$storage_name'\n" if !defined($scfg);
7607
7608     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7609         return 1;
7610     }
7611
7612     if ($volid =~ m/\.(qcow2|qed)$/){
7613         return 1;
7614     }
7615
7616     return;
7617 }
7618
7619 sub qga_check_running {
7620     my ($vmid, $nowarn) = @_;
7621
7622     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7623     if ($@) {
7624         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7625         return 0;
7626     }
7627     return 1;
7628 }
7629
7630 sub template_create {
7631     my ($vmid, $conf, $disk) = @_;
7632
7633     my $storecfg = PVE::Storage::config();
7634
7635     PVE::QemuConfig->foreach_volume($conf, sub {
7636         my ($ds, $drive) = @_;
7637
7638         return if drive_is_cdrom($drive);
7639         return if $disk && $ds ne $disk;
7640
7641         my $volid = $drive->{file};
7642         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7643
7644         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7645         $drive->{file} = $voliddst;
7646         $conf->{$ds} = print_drive($drive);
7647         PVE::QemuConfig->write_config($vmid, $conf);
7648     });
7649 }
7650
7651 sub convert_iscsi_path {
7652     my ($path) = @_;
7653
7654     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7655         my $portal = $1;
7656         my $target = $2;
7657         my $lun = $3;
7658
7659         my $initiator_name = get_initiator_name();
7660
7661         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7662                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7663     }
7664
7665     die "cannot convert iscsi path '$path', unkown format\n";
7666 }
7667
7668 sub qemu_img_convert {
7669     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7670
7671     my $storecfg = PVE::Storage::config();
7672     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7673     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7674
7675     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7676
7677     my $cachemode;
7678     my $src_path;
7679     my $src_is_iscsi = 0;
7680     my $src_format;
7681
7682     if ($src_storeid) {
7683         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7684         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7685         $src_format = qemu_img_format($src_scfg, $src_volname);
7686         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7687         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7688         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7689     } elsif (-f $src_volid || -b $src_volid) {
7690         $src_path = $src_volid;
7691         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7692             $src_format = $1;
7693         }
7694     }
7695
7696     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7697
7698     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7699     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7700     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7701     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7702
7703     my $cmd = [];
7704     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7705     push @$cmd, '-l', "snapshot.name=$snapname"
7706         if $snapname && $src_format && $src_format eq "qcow2";
7707     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7708     push @$cmd, '-T', $cachemode if defined($cachemode);
7709     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7710
7711     if ($src_is_iscsi) {
7712         push @$cmd, '--image-opts';
7713         $src_path = convert_iscsi_path($src_path);
7714     } elsif ($src_format) {
7715         push @$cmd, '-f', $src_format;
7716     }
7717
7718     if ($dst_is_iscsi) {
7719         push @$cmd, '--target-image-opts';
7720         $dst_path = convert_iscsi_path($dst_path);
7721     } else {
7722         push @$cmd, '-O', $dst_format;
7723     }
7724
7725     push @$cmd, $src_path;
7726
7727     if (!$dst_is_iscsi && $is_zero_initialized) {
7728         push @$cmd, "zeroinit:$dst_path";
7729     } else {
7730         push @$cmd, $dst_path;
7731     }
7732
7733     my $parser = sub {
7734         my $line = shift;
7735         if($line =~ m/\((\S+)\/100\%\)/){
7736             my $percent = $1;
7737             my $transferred = int($size * $percent / 100);
7738             my $total_h = render_bytes($size, 1);
7739             my $transferred_h = render_bytes($transferred, 1);
7740
7741             print "transferred $transferred_h of $total_h ($percent%)\n";
7742         }
7743
7744     };
7745
7746     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7747     my $err = $@;
7748     die "copy failed: $err" if $err;
7749 }
7750
7751 sub qemu_img_format {
7752     my ($scfg, $volname) = @_;
7753
7754     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7755         return $1;
7756     } else {
7757         return "raw";
7758     }
7759 }
7760
7761 sub qemu_drive_mirror {
7762     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7763
7764     $jobs = {} if !$jobs;
7765
7766     my $qemu_target;
7767     my $format;
7768     $jobs->{"drive-$drive"} = {};
7769
7770     if ($dst_volid =~ /^nbd:/) {
7771         $qemu_target = $dst_volid;
7772         $format = "nbd";
7773     } else {
7774         my $storecfg = PVE::Storage::config();
7775         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7776
7777         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7778
7779         $format = qemu_img_format($dst_scfg, $dst_volname);
7780
7781         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7782
7783         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7784     }
7785
7786     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7787     $opts->{format} = $format if $format;
7788
7789     if (defined($src_bitmap)) {
7790         $opts->{sync} = 'incremental';
7791         $opts->{bitmap} = $src_bitmap;
7792         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7793     }
7794
7795     if (defined($bwlimit)) {
7796         $opts->{speed} = $bwlimit * 1024;
7797         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7798     } else {
7799         print "drive mirror is starting for drive-$drive\n";
7800     }
7801
7802     # if a job already runs for this device we get an error, catch it for cleanup
7803     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7804     if (my $err = $@) {
7805         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7806         warn "$@\n" if $@;
7807         die "mirroring error: $err\n";
7808     }
7809
7810     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7811 }
7812
7813 # $completion can be either
7814 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7815 # 'cancel': wait until all jobs are ready, block-job-cancel them
7816 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7817 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7818 sub qemu_drive_mirror_monitor {
7819     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7820
7821     $completion //= 'complete';
7822     $op //= "mirror";
7823
7824     eval {
7825         my $err_complete = 0;
7826
7827         my $starttime = time ();
7828         while (1) {
7829             die "block job ('$op') timed out\n" if $err_complete > 300;
7830
7831             my $stats = mon_cmd($vmid, "query-block-jobs");
7832             my $ctime = time();
7833
7834             my $running_jobs = {};
7835             for my $stat (@$stats) {
7836                 next if $stat->{type} ne $op;
7837                 $running_jobs->{$stat->{device}} = $stat;
7838             }
7839
7840             my $readycounter = 0;
7841
7842             for my $job_id (sort keys %$jobs) {
7843                 my $job = $running_jobs->{$job_id};
7844
7845                 my $vanished = !defined($job);
7846                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7847                 if($complete || ($vanished && $completion eq 'auto')) {
7848                     print "$job_id: $op-job finished\n";
7849                     delete $jobs->{$job_id};
7850                     next;
7851                 }
7852
7853                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7854
7855                 my $busy = $job->{busy};
7856                 my $ready = $job->{ready};
7857                 if (my $total = $job->{len}) {
7858                     my $transferred = $job->{offset} || 0;
7859                     my $remaining = $total - $transferred;
7860                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7861
7862                     my $duration = $ctime - $starttime;
7863                     my $total_h = render_bytes($total, 1);
7864                     my $transferred_h = render_bytes($transferred, 1);
7865
7866                     my $status = sprintf(
7867                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7868                         render_duration($duration),
7869                     );
7870
7871                     if ($ready) {
7872                         if ($busy) {
7873                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7874                         } else {
7875                             $status .= ", ready";
7876                         }
7877                     }
7878                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7879                     $jobs->{$job_id}->{ready} = $ready;
7880                 }
7881
7882                 $readycounter++ if $job->{ready};
7883             }
7884
7885             last if scalar(keys %$jobs) == 0;
7886
7887             if ($readycounter == scalar(keys %$jobs)) {
7888                 print "all '$op' jobs are ready\n";
7889
7890                 # do the complete later (or has already been done)
7891                 last if $completion eq 'skip' || $completion eq 'auto';
7892
7893                 if ($vmiddst && $vmiddst != $vmid) {
7894                     my $agent_running = $qga && qga_check_running($vmid);
7895                     if ($agent_running) {
7896                         print "freeze filesystem\n";
7897                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7898                         warn $@ if $@;
7899                     } else {
7900                         print "suspend vm\n";
7901                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7902                         warn $@ if $@;
7903                     }
7904
7905                     # if we clone a disk for a new target vm, we don't switch the disk
7906                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7907
7908                     if ($agent_running) {
7909                         print "unfreeze filesystem\n";
7910                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7911                         warn $@ if $@;
7912                     } else {
7913                         print "resume vm\n";
7914                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7915                         warn $@ if $@;
7916                     }
7917
7918                     last;
7919                 } else {
7920
7921                     for my $job_id (sort keys %$jobs) {
7922                         # try to switch the disk if source and destination are on the same guest
7923                         print "$job_id: Completing block job_id...\n";
7924
7925                         my $op;
7926                         if ($completion eq 'complete') {
7927                             $op = 'block-job-complete';
7928                         } elsif ($completion eq 'cancel') {
7929                             $op = 'block-job-cancel';
7930                         } else {
7931                             die "invalid completion value: $completion\n";
7932                         }
7933                         eval { mon_cmd($vmid, $op, device => $job_id) };
7934                         if ($@ =~ m/cannot be completed/) {
7935                             print "$job_id: block job cannot be completed, trying again.\n";
7936                             $err_complete++;
7937                         }else {
7938                             print "$job_id: Completed successfully.\n";
7939                             $jobs->{$job_id}->{complete} = 1;
7940                         }
7941                     }
7942                 }
7943             }
7944             sleep 1;
7945         }
7946     };
7947     my $err = $@;
7948
7949     if ($err) {
7950         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7951         die "block job ($op) error: $err";
7952     }
7953 }
7954
7955 sub qemu_blockjobs_cancel {
7956     my ($vmid, $jobs) = @_;
7957
7958     foreach my $job (keys %$jobs) {
7959         print "$job: Cancelling block job\n";
7960         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7961         $jobs->{$job}->{cancel} = 1;
7962     }
7963
7964     while (1) {
7965         my $stats = mon_cmd($vmid, "query-block-jobs");
7966
7967         my $running_jobs = {};
7968         foreach my $stat (@$stats) {
7969             $running_jobs->{$stat->{device}} = $stat;
7970         }
7971
7972         foreach my $job (keys %$jobs) {
7973
7974             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7975                 print "$job: Done.\n";
7976                 delete $jobs->{$job};
7977             }
7978         }
7979
7980         last if scalar(keys %$jobs) == 0;
7981
7982         sleep 1;
7983     }
7984 }
7985
7986 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
7987 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
7988 my sub clone_disk_check_io_uring {
7989     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
7990
7991     return if !$use_drive_mirror;
7992
7993     # Don't complain when not changing storage.
7994     # Assume if it works for the source, it'll work for the target too.
7995     return if $src_storeid eq $dst_storeid;
7996
7997     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7998     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7999
8000     my $cache_direct = drive_uses_cache_direct($src_drive);
8001
8002     my $src_uses_io_uring;
8003     if ($src_drive->{aio}) {
8004         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8005     } else {
8006         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8007     }
8008
8009     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8010         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8011 }
8012
8013 sub clone_disk {
8014     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8015
8016     my ($vmid, $running) = $source->@{qw(vmid running)};
8017     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8018
8019     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8020     my ($storage, $format) = $dest->@{qw(storage format)};
8021
8022     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8023
8024     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8025         die "cloning from/to EFI disk requires EFI disk\n"
8026             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8027         die "cloning from/to TPM state requires TPM state\n"
8028             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8029
8030         # This would lead to two device nodes in QEMU pointing to the same backing image!
8031         die "cannot change drive name when cloning disk from/to the same VM\n"
8032             if $use_drive_mirror && $vmid == $newvmid;
8033     }
8034
8035     die "cannot move TPM state while VM is running\n"
8036         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8037
8038     my $newvolid;
8039
8040     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8041     print "$src_drivename " if $src_drivename;
8042     print "($drive->{file})\n";
8043
8044     if (!$full) {
8045         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8046         push @$newvollist, $newvolid;
8047     } else {
8048         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8049         my $storeid = $storage || $src_storeid;
8050
8051         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8052
8053         my $name = undef;
8054         my $size = undef;
8055         if (drive_is_cloudinit($drive)) {
8056             $name = "vm-$newvmid-cloudinit";
8057             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8058             if ($scfg->{path}) {
8059                 $name .= ".$dst_format";
8060             }
8061             $snapname = undef;
8062             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8063         } elsif ($dst_drivename eq 'efidisk0') {
8064             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8065         } elsif ($dst_drivename eq 'tpmstate0') {
8066             $dst_format = 'raw';
8067             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8068         } else {
8069             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8070
8071             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8072         }
8073         $newvolid = PVE::Storage::vdisk_alloc(
8074             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8075         );
8076         push @$newvollist, $newvolid;
8077
8078         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8079
8080         if (drive_is_cloudinit($drive)) {
8081             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8082             # if this is the case, we have to complete any block-jobs still there from
8083             # previous drive-mirrors
8084             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8085                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8086             }
8087             goto no_data_clone;
8088         }
8089
8090         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8091         if ($use_drive_mirror) {
8092             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8093                 $completion, $qga, $bwlimit);
8094         } else {
8095             if ($dst_drivename eq 'efidisk0') {
8096                 # the relevant data on the efidisk may be smaller than the source
8097                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8098                 # that is given by the OVMF_VARS.fd
8099                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8100                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8101
8102                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8103
8104                 # better for Ceph if block size is not too small, see bug #3324
8105                 my $bs = 1024*1024;
8106
8107                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8108
8109                 if ($src_format eq 'qcow2' && $snapname) {
8110                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8111                         if !min_version(kvm_user_version(), 6, 2);
8112                     push $cmd->@*, '-l', $snapname;
8113                 }
8114                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8115                 run_command($cmd);
8116             } else {
8117                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8118             }
8119         }
8120     }
8121
8122 no_data_clone:
8123     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8124
8125     my $disk = dclone($drive);
8126     delete $disk->{format};
8127     $disk->{file} = $newvolid;
8128     $disk->{size} = $size if defined($size);
8129
8130     return $disk;
8131 }
8132
8133 sub get_running_qemu_version {
8134     my ($vmid) = @_;
8135     my $res = mon_cmd($vmid, "query-version");
8136     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8137 }
8138
8139 sub qemu_use_old_bios_files {
8140     my ($machine_type) = @_;
8141
8142     return if !$machine_type;
8143
8144     my $use_old_bios_files = undef;
8145
8146     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8147         $machine_type = $1;
8148         $use_old_bios_files = 1;
8149     } else {
8150         my $version = extract_version($machine_type, kvm_user_version());
8151         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8152         # load new efi bios files on migration. So this hack is required to allow
8153         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8154         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8155         $use_old_bios_files = !min_version($version, 2, 4);
8156     }
8157
8158     return ($use_old_bios_files, $machine_type);
8159 }
8160
8161 sub get_efivars_size {
8162     my ($conf, $efidisk) = @_;
8163
8164     my $arch = get_vm_arch($conf);
8165     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8166     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8167     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8168     return -s $ovmf_vars;
8169 }
8170
8171 sub update_efidisk_size {
8172     my ($conf) = @_;
8173
8174     return if !defined($conf->{efidisk0});
8175
8176     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8177     $disk->{size} = get_efivars_size($conf);
8178     $conf->{efidisk0} = print_drive($disk);
8179
8180     return;
8181 }
8182
8183 sub update_tpmstate_size {
8184     my ($conf) = @_;
8185
8186     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8187     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8188     $conf->{tpmstate0} = print_drive($disk);
8189 }
8190
8191 sub create_efidisk($$$$$$$) {
8192     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8193
8194     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8195
8196     my $vars_size_b = -s $ovmf_vars;
8197     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8198     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8199     PVE::Storage::activate_volumes($storecfg, [$volid]);
8200
8201     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8202     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8203
8204     return ($volid, $size/1024);
8205 }
8206
8207 sub vm_iothreads_list {
8208     my ($vmid) = @_;
8209
8210     my $res = mon_cmd($vmid, 'query-iothreads');
8211
8212     my $iothreads = {};
8213     foreach my $iothread (@$res) {
8214         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8215     }
8216
8217     return $iothreads;
8218 }
8219
8220 sub scsihw_infos {
8221     my ($conf, $drive) = @_;
8222
8223     my $maxdev = 0;
8224
8225     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8226         $maxdev = 7;
8227     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8228         $maxdev = 1;
8229     } else {
8230         $maxdev = 256;
8231     }
8232
8233     my $controller = int($drive->{index} / $maxdev);
8234     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8235         ? "virtioscsi"
8236         : "scsihw";
8237
8238     return ($maxdev, $controller, $controller_prefix);
8239 }
8240
8241 sub resolve_dst_disk_format {
8242         my ($storecfg, $storeid, $src_volname, $format) = @_;
8243         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8244
8245         if (!$format) {
8246             # if no target format is specified, use the source disk format as hint
8247             if ($src_volname) {
8248                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8249                 $format = qemu_img_format($scfg, $src_volname);
8250             } else {
8251                 return $defFormat;
8252             }
8253         }
8254
8255         # test if requested format is supported - else use default
8256         my $supported = grep { $_ eq $format } @$validFormats;
8257         $format = $defFormat if !$supported;
8258         return $format;
8259 }
8260
8261 # NOTE: if this logic changes, please update docs & possibly gui logic
8262 sub find_vmstate_storage {
8263     my ($conf, $storecfg) = @_;
8264
8265     # first, return storage from conf if set
8266     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8267
8268     my ($target, $shared, $local);
8269
8270     foreach_storage_used_by_vm($conf, sub {
8271         my ($sid) = @_;
8272         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8273         my $dst = $scfg->{shared} ? \$shared : \$local;
8274         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8275     });
8276
8277     # second, use shared storage where VM has at least one disk
8278     # third, use local storage where VM has at least one disk
8279     # fall back to local storage
8280     $target = $shared // $local // 'local';
8281
8282     return $target;
8283 }
8284
8285 sub generate_uuid {
8286     my ($uuid, $uuid_str);
8287     UUID::generate($uuid);
8288     UUID::unparse($uuid, $uuid_str);
8289     return $uuid_str;
8290 }
8291
8292 sub generate_smbios1_uuid {
8293     return "uuid=".generate_uuid();
8294 }
8295
8296 sub nbd_stop {
8297     my ($vmid) = @_;
8298
8299     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8300 }
8301
8302 sub create_reboot_request {
8303     my ($vmid) = @_;
8304     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8305         or die "failed to create reboot trigger file: $!\n";
8306     close($fh);
8307 }
8308
8309 sub clear_reboot_request {
8310     my ($vmid) = @_;
8311     my $path = "/run/qemu-server/$vmid.reboot";
8312     my $res = 0;
8313
8314     $res = unlink($path);
8315     die "could not remove reboot request for $vmid: $!"
8316         if !$res && $! != POSIX::ENOENT;
8317
8318     return $res;
8319 }
8320
8321 sub bootorder_from_legacy {
8322     my ($conf, $bootcfg) = @_;
8323
8324     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8325     my $bootindex_hash = {};
8326     my $i = 1;
8327     foreach my $o (split(//, $boot)) {
8328         $bootindex_hash->{$o} = $i*100;
8329         $i++;
8330     }
8331
8332     my $bootorder = {};
8333
8334     PVE::QemuConfig->foreach_volume($conf, sub {
8335         my ($ds, $drive) = @_;
8336
8337         if (drive_is_cdrom ($drive, 1)) {
8338             if ($bootindex_hash->{d}) {
8339                 $bootorder->{$ds} = $bootindex_hash->{d};
8340                 $bootindex_hash->{d} += 1;
8341             }
8342         } elsif ($bootindex_hash->{c}) {
8343             $bootorder->{$ds} = $bootindex_hash->{c}
8344                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8345             $bootindex_hash->{c} += 1;
8346         }
8347     });
8348
8349     if ($bootindex_hash->{n}) {
8350         for (my $i = 0; $i < $MAX_NETS; $i++) {
8351             my $netname = "net$i";
8352             next if !$conf->{$netname};
8353             $bootorder->{$netname} = $bootindex_hash->{n};
8354             $bootindex_hash->{n} += 1;
8355         }
8356     }
8357
8358     return $bootorder;
8359 }
8360
8361 # Generate default device list for 'boot: order=' property. Matches legacy
8362 # default boot order, but with explicit device names. This is important, since
8363 # the fallback for when neither 'order' nor the old format is specified relies
8364 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8365 sub get_default_bootdevices {
8366     my ($conf) = @_;
8367
8368     my @ret = ();
8369
8370     # harddisk
8371     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8372     push @ret, $first if $first;
8373
8374     # cdrom
8375     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8376     push @ret, $first if $first;
8377
8378     # network
8379     for (my $i = 0; $i < $MAX_NETS; $i++) {
8380         my $netname = "net$i";
8381         next if !$conf->{$netname};
8382         push @ret, $netname;
8383         last;
8384     }
8385
8386     return \@ret;
8387 }
8388
8389 sub device_bootorder {
8390     my ($conf) = @_;
8391
8392     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8393
8394     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8395
8396     my $bootorder = {};
8397     if (!defined($boot) || $boot->{legacy}) {
8398         $bootorder = bootorder_from_legacy($conf, $boot);
8399     } elsif ($boot->{order}) {
8400         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8401         for my $dev (PVE::Tools::split_list($boot->{order})) {
8402             $bootorder->{$dev} = $i++;
8403         }
8404     }
8405
8406     return $bootorder;
8407 }
8408
8409 sub register_qmeventd_handle {
8410     my ($vmid) = @_;
8411
8412     my $fh;
8413     my $peer = "/var/run/qmeventd.sock";
8414     my $count = 0;
8415
8416     for (;;) {
8417         $count++;
8418         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8419         last if $fh;
8420         if ($! != EINTR && $! != EAGAIN) {
8421             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8422         }
8423         if ($count > 4) {
8424             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8425               . "after $count retries\n";
8426         }
8427         usleep(25000);
8428     }
8429
8430     # send handshake to mark VM as backing up
8431     print $fh to_json({vzdump => {vmid => "$vmid"}});
8432
8433     # return handle to be closed later when inhibit is no longer required
8434     return $fh;
8435 }
8436
8437 # bash completion helper
8438
8439 sub complete_backup_archives {
8440     my ($cmdname, $pname, $cvalue) = @_;
8441
8442     my $cfg = PVE::Storage::config();
8443
8444     my $storeid;
8445
8446     if ($cvalue =~ m/^([^:]+):/) {
8447         $storeid = $1;
8448     }
8449
8450     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8451
8452     my $res = [];
8453     foreach my $id (keys %$data) {
8454         foreach my $item (@{$data->{$id}}) {
8455             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8456             push @$res, $item->{volid} if defined($item->{volid});
8457         }
8458     }
8459
8460     return $res;
8461 }
8462
8463 my $complete_vmid_full = sub {
8464     my ($running) = @_;
8465
8466     my $idlist = vmstatus();
8467
8468     my $res = [];
8469
8470     foreach my $id (keys %$idlist) {
8471         my $d = $idlist->{$id};
8472         if (defined($running)) {
8473             next if $d->{template};
8474             next if $running && $d->{status} ne 'running';
8475             next if !$running && $d->{status} eq 'running';
8476         }
8477         push @$res, $id;
8478
8479     }
8480     return $res;
8481 };
8482
8483 sub complete_vmid {
8484     return &$complete_vmid_full();
8485 }
8486
8487 sub complete_vmid_stopped {
8488     return &$complete_vmid_full(0);
8489 }
8490
8491 sub complete_vmid_running {
8492     return &$complete_vmid_full(1);
8493 }
8494
8495 sub complete_storage {
8496
8497     my $cfg = PVE::Storage::config();
8498     my $ids = $cfg->{ids};
8499
8500     my $res = [];
8501     foreach my $sid (keys %$ids) {
8502         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8503         next if !$ids->{$sid}->{content}->{images};
8504         push @$res, $sid;
8505     }
8506
8507     return $res;
8508 }
8509
8510 sub complete_migration_storage {
8511     my ($cmd, $param, $current_value, $all_args) = @_;
8512
8513     my $targetnode = @$all_args[1];
8514
8515     my $cfg = PVE::Storage::config();
8516     my $ids = $cfg->{ids};
8517
8518     my $res = [];
8519     foreach my $sid (keys %$ids) {
8520         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8521         next if !$ids->{$sid}->{content}->{images};
8522         push @$res, $sid;
8523     }
8524
8525     return $res;
8526 }
8527
8528 sub vm_is_paused {
8529     my ($vmid, $include_suspended) = @_;
8530     my $qmpstatus = eval {
8531         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8532         mon_cmd($vmid, "query-status");
8533     };
8534     warn "$@\n" if $@;
8535     return $qmpstatus && (
8536         $qmpstatus->{status} eq "paused" ||
8537         $qmpstatus->{status} eq "prelaunch" ||
8538         ($include_suspended && $qmpstatus->{status} eq "suspended")
8539     );
8540 }
8541
8542 sub check_volume_storage_type {
8543     my ($storecfg, $vol) = @_;
8544
8545     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8546     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8547     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8548
8549     die "storage '$storeid' does not support content-type '$vtype'\n"
8550         if !$scfg->{content}->{$vtype};
8551
8552     return 1;
8553 }
8554
8555 sub add_nets_bridge_fdb {
8556     my ($conf, $vmid) = @_;
8557
8558     for my $opt (keys %$conf) {
8559         next if $opt !~ m/^net(\d+)$/;
8560         my $iface = "tap${vmid}i$1";
8561         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8562         my $net = parse_net($conf->{$opt}, 1) or next;
8563
8564         my $mac = $net->{macaddr};
8565         if (!$mac) {
8566             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8567                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8568             next;
8569         }
8570
8571         my $bridge = $net->{bridge};
8572         if (!$bridge) {
8573             log_warn("Interface '$iface' not attached to any bridge.");
8574             next;
8575         }
8576         if ($have_sdn) {
8577             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8578         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8579             PVE::Network::add_bridge_fdb($iface, $mac);
8580         }
8581     }
8582 }
8583
8584 sub del_nets_bridge_fdb {
8585     my ($conf, $vmid) = @_;
8586
8587     for my $opt (keys %$conf) {
8588         next if $opt !~ m/^net(\d+)$/;
8589         my $iface = "tap${vmid}i$1";
8590
8591         my $net = parse_net($conf->{$opt}) or next;
8592         my $mac = $net->{macaddr} or next;
8593
8594         my $bridge = $net->{bridge};
8595         if ($have_sdn) {
8596             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8597         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8598             PVE::Network::del_bridge_fdb($iface, $mac);
8599         }
8600     }
8601 }
8602
8603 sub create_ifaces_ipams_ips {
8604     my ($conf, $vmid) = @_;
8605
8606     return if !$have_sdn;
8607
8608     foreach my $opt (keys %$conf) {
8609         if ($opt =~ m/^net(\d+)$/) {
8610             my $value = $conf->{$opt};
8611             my $net = PVE::QemuServer::parse_net($value);
8612             eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8613             warn $@ if $@;
8614         }
8615     }
8616 }
8617
8618 sub delete_ifaces_ipams_ips {
8619     my ($conf, $vmid) = @_;
8620
8621     return if !$have_sdn;
8622
8623     foreach my $opt (keys %$conf) {
8624         if ($opt =~ m/^net(\d+)$/) {
8625             my $net = PVE::QemuServer::parse_net($conf->{$opt});
8626             eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8627             warn $@ if $@;
8628         }
8629     }
8630 }
8631
8632 1;