PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory qw(get_current_memory);
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  62 use PVE::QemuServer::USB;
  63
  64 my $have_sdn;
  65 eval {
  66     require PVE::Network::SDN::Zones;
  67     require PVE::Network::SDN::Vnets;
  68     $have_sdn = 1;
  69 };
  70
  71 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  72 my $OVMF = {
  73     x86_64 => {
  74         '4m-no-smm' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  77         ],
  78         '4m-no-smm-ms' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  81         ],
  82         '4m' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  85         ],
  86         '4m-ms' => [
  87             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  89         ],
  90         # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
  91         # anymore. how can we deperacate this sanely without breaking existing instances, or using
  92         # older backups and snapshot?
  93         default => [
  94             "$EDK2_FW_BASE/OVMF_CODE.fd",
  95             "$EDK2_FW_BASE/OVMF_VARS.fd",
  96         ],
  97     },
  98     aarch64 => {
  99         default => [
 100             "$EDK2_FW_BASE/AAVMF_CODE.fd",
 101             "$EDK2_FW_BASE/AAVMF_VARS.fd",
 102         ],
 103     },
 104 };
 105
 106 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 107
 108 # Note about locking: we use flock on the config file protect against concurent actions.
 109 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 110 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 111 # But you can ignore this kind of lock with the --skiplock flag.
 112
 113 cfs_register_file(
 114     '/qemu-server/',
 115     \&parse_vm_config,
 116     \&write_vm_config
 117 );
 118
 119 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 120     description => "Some command save/restore state from this location.",
 121     type => 'string',
 122     maxLength => 128,
 123     optional => 1,
 124 });
 125
 126 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 127         description => "Specifies the QEMU machine type.",
 128         type => 'string',
 129         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 130         maxLength => 40,
 131         optional => 1,
 132 });
 133
 134 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 135 my $nodename_cache;
 136 sub nodename {
 137     $nodename_cache //= PVE::INotify::nodename();
 138     return $nodename_cache;
 139 }
 140
 141 my $watchdog_fmt = {
 142     model => {
 143         default_key => 1,
 144         type => 'string',
 145         enum => [qw(i6300esb ib700)],
 146         description => "Watchdog type to emulate.",
 147         default => 'i6300esb',
 148         optional => 1,
 149     },
 150     action => {
 151         type => 'string',
 152         enum => [qw(reset shutdown poweroff pause debug none)],
 153         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 154         optional => 1,
 155     },
 156 };
 157 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 158
 159 my $agent_fmt = {
 160     enabled => {
 161         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 162         type => 'boolean',
 163         default => 0,
 164         default_key => 1,
 165     },
 166     fstrim_cloned_disks => {
 167         description => "Run fstrim after moving a disk or migrating the VM.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0,
 171     },
 172     'freeze-fs-on-backup' => {
 173         description => "Freeze/thaw guest filesystems on backup for consistency.",
 174         type => 'boolean',
 175         optional => 1,
 176         default => 1,
 177     },
 178     type => {
 179         description => "Select the agent type",
 180         type => 'string',
 181         default => 'virtio',
 182         optional => 1,
 183         enum => [qw(virtio isa)],
 184     },
 185 };
 186
 187 my $vga_fmt = {
 188     type => {
 189         description => "Select the VGA type.",
 190         type => 'string',
 191         default => 'std',
 192         optional => 1,
 193         default_key => 1,
 194         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 195     },
 196     memory => {
 197         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 198         type => 'integer',
 199         optional => 1,
 200         minimum => 4,
 201         maximum => 512,
 202     },
 203     clipboard => {
 204         description => 'Enable a specific clipboard. If not set, depending on the display type the'
 205             .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
 206         type => 'string',
 207         enum => ['vnc'],
 208         optional => 1,
 209     },
 210 };
 211
 212 my $ivshmem_fmt = {
 213     size => {
 214         type => 'integer',
 215         minimum => 1,
 216         description => "The size of the file in MB.",
 217     },
 218     name => {
 219         type => 'string',
 220         pattern => '[a-zA-Z0-9\-]+',
 221         optional => 1,
 222         format_description => 'string',
 223         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 224     },
 225 };
 226
 227 my $audio_fmt = {
 228     device => {
 229         type => 'string',
 230         enum => [qw(ich9-intel-hda intel-hda AC97)],
 231         description =>  "Configure an audio device."
 232     },
 233     driver =>  {
 234         type => 'string',
 235         enum => ['spice', 'none'],
 236         default => 'spice',
 237         optional => 1,
 238         description => "Driver backend for the audio device."
 239     },
 240 };
 241
 242 my $spice_enhancements_fmt = {
 243     foldersharing => {
 244         type => 'boolean',
 245         optional => 1,
 246         default => '0',
 247         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 248     },
 249     videostreaming =>  {
 250         type => 'string',
 251         enum => ['off', 'all', 'filter'],
 252         default => 'off',
 253         optional => 1,
 254         description => "Enable video streaming. Uses compression for detected video streams."
 255     },
 256 };
 257
 258 my $rng_fmt = {
 259     source => {
 260         type => 'string',
 261         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 262         default_key => 1,
 263         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 264             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 265             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 266             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 267             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 268             ." a hardware RNG from the host.",
 269     },
 270     max_bytes => {
 271         type => 'integer',
 272         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 273             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 274             ." `0` to disable limiting (potentially dangerous!).",
 275         optional => 1,
 276
 277         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 278         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 279         # reading from /dev/urandom
 280         default => 1024,
 281     },
 282     period => {
 283         type => 'integer',
 284         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 285             ." the guest to retrieve another 'max_bytes' of entropy.",
 286         optional => 1,
 287         default => 1000,
 288     },
 289 };
 290
 291 my $meta_info_fmt = {
 292     'ctime' => {
 293         type => 'integer',
 294         description => "The guest creation timestamp as UNIX epoch time",
 295         minimum => 0,
 296         optional => 1,
 297     },
 298     'creation-qemu' => {
 299         type => 'string',
 300         description => "The QEMU (machine) version from the time this VM was created.",
 301         pattern => '\d+(\.\d+)+',
 302         optional => 1,
 303     },
 304 };
 305
 306 my $confdesc = {
 307     onboot => {
 308         optional => 1,
 309         type => 'boolean',
 310         description => "Specifies whether a VM will be started during system bootup.",
 311         default => 0,
 312     },
 313     autostart => {
 314         optional => 1,
 315         type => 'boolean',
 316         description => "Automatic restart after crash (currently ignored).",
 317         default => 0,
 318     },
 319     hotplug => {
 320         optional => 1,
 321         type => 'string', format => 'pve-hotplug-features',
 322         description => "Selectively enable hotplug features. This is a comma separated list of"
 323             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 324             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 325             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 326             ." windows > 7.",
 327         default => 'network,disk,usb',
 328     },
 329     reboot => {
 330         optional => 1,
 331         type => 'boolean',
 332         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 333         default => 1,
 334     },
 335     lock => {
 336         optional => 1,
 337         type => 'string',
 338         description => "Lock/unlock the VM.",
 339         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 340     },
 341     cpulimit => {
 342         optional => 1,
 343         type => 'number',
 344         description => "Limit of CPU usage.",
 345         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 346             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 347         minimum => 0,
 348         maximum => 128,
 349         default => 0,
 350     },
 351     cpuunits => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 355         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 356             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 357             ." weights of all the other running VMs.",
 358         minimum => 1,
 359         maximum => 262144,
 360         default => 'cgroup v1: 1024, cgroup v2: 100',
 361     },
 362     memory => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Memory properties.",
 366         format => $PVE::QemuServer::Memory::memory_fmt
 367     },
 368     balloon => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 372         minimum => 0,
 373     },
 374     shares => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 378             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 379             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 380         minimum => 0,
 381         maximum => 50000,
 382         default => 1000,
 383     },
 384     keyboard => {
 385         optional => 1,
 386         type => 'string',
 387         description => "Keyboard layout for VNC server. This option is generally not required and"
 388         ." is often better handled from within the guest OS.",
 389         enum => PVE::Tools::kvmkeymaplist(),
 390         default => undef,
 391     },
 392     name => {
 393         optional => 1,
 394         type => 'string', format => 'dns-name',
 395         description => "Set a name for the VM. Only used on the configuration web interface.",
 396     },
 397     scsihw => {
 398         optional => 1,
 399         type => 'string',
 400         description => "SCSI controller model",
 401         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 402         default => 'lsi',
 403     },
 404     description => {
 405         optional => 1,
 406         type => 'string',
 407         description => "Description for the VM. Shown in the web-interface VM's summary."
 408             ." This is saved as comment inside the configuration file.",
 409         maxLength => 1024 * 8,
 410     },
 411     ostype => {
 412         optional => 1,
 413         type => 'string',
 414         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 415         description => "Specify guest operating system.",
 416         verbose_description => <<EODESC,
 417 Specify guest operating system. This is used to enable special
 418 optimization/features for specific operating systems:
 419
 420 [horizontal]
 421 other;; unspecified OS
 422 wxp;; Microsoft Windows XP
 423 w2k;; Microsoft Windows 2000
 424 w2k3;; Microsoft Windows 2003
 425 w2k8;; Microsoft Windows 2008
 426 wvista;; Microsoft Windows Vista
 427 win7;; Microsoft Windows 7
 428 win8;; Microsoft Windows 8/2012/2012r2
 429 win10;; Microsoft Windows 10/2016/2019
 430 win11;; Microsoft Windows 11/2022
 431 l24;; Linux 2.4 Kernel
 432 l26;; Linux 2.6 - 6.X Kernel
 433 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 434 EODESC
 435     },
 436     boot => {
 437         optional => 1,
 438         type => 'string', format => 'pve-qm-boot',
 439         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 440             ." key or 'legacy=' is deprecated.",
 441     },
 442     bootdisk => {
 443         optional => 1,
 444         type => 'string', format => 'pve-qm-bootdisk',
 445         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 446         pattern => '(ide|sata|scsi|virtio)\d+',
 447     },
 448     smp => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "The number of CPUs. Please use option -sockets instead.",
 452         minimum => 1,
 453         default => 1,
 454     },
 455     sockets => {
 456         optional => 1,
 457         type => 'integer',
 458         description => "The number of CPU sockets.",
 459         minimum => 1,
 460         default => 1,
 461     },
 462     cores => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "The number of cores per socket.",
 466         minimum => 1,
 467         default => 1,
 468     },
 469     numa => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable NUMA.",
 473         default => 0,
 474     },
 475     hugepages => {
 476         optional => 1,
 477         type => 'string',
 478         description => "Enable/disable hugepages memory.",
 479         enum => [qw(any 2 1024)],
 480     },
 481     keephugepages => {
 482         optional => 1,
 483         type => 'boolean',
 484         default => 0,
 485         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 486             ." after VM shutdown and can be used for subsequent starts.",
 487     },
 488     vcpus => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Number of hotplugged vcpus.",
 492         minimum => 1,
 493         default => 0,
 494     },
 495     acpi => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable ACPI.",
 499         default => 1,
 500     },
 501     agent => {
 502         optional => 1,
 503         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 504         type => 'string',
 505         format => $agent_fmt,
 506     },
 507     kvm => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable KVM hardware virtualization.",
 511         default => 1,
 512     },
 513     tdf => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable time drift fix.",
 517         default => 0,
 518     },
 519     localtime => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 523             ." the `ostype` indicates a Microsoft Windows OS.",
 524     },
 525     freeze => {
 526         optional => 1,
 527         type => 'boolean',
 528         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 529     },
 530     vga => {
 531         optional => 1,
 532         type => 'string', format => $vga_fmt,
 533         description => "Configure the VGA hardware.",
 534         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 535             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 536             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 537             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 538             ." display server. For win* OS you can select how many independent displays you want,"
 539             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 540             ." using a serial device as terminal.",
 541     },
 542     watchdog => {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-watchdog',
 545         description => "Create a virtual hardware watchdog device.",
 546         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 547             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 548             ." else the watchdog will reset the guest (or execute the respective action specified)",
 549     },
 550     startdate => {
 551         optional => 1,
 552         type => 'string',
 553         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 554         description => "Set the initial date of the real time clock. Valid format for date are:"
 555             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 556         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 557         default => 'now',
 558     },
 559     startup =>  get_standard_option('pve-startup-order'),
 560     template => {
 561         optional => 1,
 562         type => 'boolean',
 563         description => "Enable/disable Template.",
 564         default => 0,
 565     },
 566     args => {
 567         optional => 1,
 568         type => 'string',
 569         description => "Arbitrary arguments passed to kvm.",
 570         verbose_description => <<EODESCR,
 571 Arbitrary arguments passed to kvm, for example:
 572
 573 args: -no-reboot -smbios 'type=0,vendor=FOO'
 574
 575 NOTE: this option is for experts only.
 576 EODESCR
 577     },
 578     tablet => {
 579         optional => 1,
 580         type => 'boolean',
 581         default => 1,
 582         description => "Enable/disable the USB tablet device.",
 583         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 584             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 585             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 586             ." may consider disabling this to save some context switches. This is turned off by"
 587             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 588     },
 589     migrate_speed => {
 590         optional => 1,
 591         type => 'integer',
 592         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 593         minimum => 0,
 594         default => 0,
 595     },
 596     migrate_downtime => {
 597         optional => 1,
 598         type => 'number',
 599         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 600         minimum => 0,
 601         default => 0.1,
 602     },
 603     cdrom => {
 604         optional => 1,
 605         type => 'string', format => 'pve-qm-ide',
 606         typetext => '<volume>',
 607         description => "This is an alias for option -ide2",
 608     },
 609     cpu => {
 610         optional => 1,
 611         description => "Emulated CPU type.",
 612         type => 'string',
 613         format => 'pve-vm-cpu-conf',
 614     },
 615     parent => get_standard_option('pve-snapshot-name', {
 616         optional => 1,
 617         description => "Parent snapshot name. This is used internally, and should not be modified.",
 618     }),
 619     snaptime => {
 620         optional => 1,
 621         description => "Timestamp for snapshots.",
 622         type => 'integer',
 623         minimum => 0,
 624     },
 625     vmstate => {
 626         optional => 1,
 627         type => 'string', format => 'pve-volume-id',
 628         description => "Reference to a volume which stores the VM state. This is used internally"
 629             ." for snapshots.",
 630     },
 631     vmstatestorage => get_standard_option('pve-storage-id', {
 632         description => "Default storage for VM state volumes/files.",
 633         optional => 1,
 634     }),
 635     runningmachine => get_standard_option('pve-qemu-machine', {
 636         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 637             ." for snapshots.",
 638     }),
 639     runningcpu => {
 640         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 641             ." internally for snapshots.",
 642         optional => 1,
 643         type => 'string',
 644         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 645         format_description => 'QEMU -cpu parameter'
 646     },
 647     machine => get_standard_option('pve-qemu-machine'),
 648     arch => {
 649         description => "Virtual processor architecture. Defaults to the host.",
 650         optional => 1,
 651         type => 'string',
 652         enum => [qw(x86_64 aarch64)],
 653     },
 654     smbios1 => {
 655         description => "Specify SMBIOS type 1 fields.",
 656         type => 'string', format => 'pve-qm-smbios1',
 657         maxLength => 512,
 658         optional => 1,
 659     },
 660     protection => {
 661         optional => 1,
 662         type => 'boolean',
 663         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 664             ." remove disk operations.",
 665         default => 0,
 666     },
 667     bios => {
 668         optional => 1,
 669         type => 'string',
 670         enum => [ qw(seabios ovmf) ],
 671         description => "Select BIOS implementation.",
 672         default => 'seabios',
 673     },
 674     vmgenid => {
 675         type => 'string',
 676         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 677         format_description => 'UUID',
 678         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 679             ." to disable explicitly.",
 680         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 681             ." value identifier to the guest OS. This allows to notify the guest operating system"
 682             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 683             ." execution or creation from a template). The guest operating system notices the"
 684             ." change, and is then able to react as appropriate by marking its copies of"
 685             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 686             ."Note that auto-creation only works when done through API/CLI create or update methods"
 687             .", but not when manually editing the config file.",
 688         default => "1 (autogenerated)",
 689         optional => 1,
 690     },
 691     hookscript => {
 692         type => 'string',
 693         format => 'pve-volume-id',
 694         optional => 1,
 695         description => "Script that will be executed during various steps in the vms lifetime.",
 696     },
 697     ivshmem => {
 698         type => 'string',
 699         format => $ivshmem_fmt,
 700         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 701             ." the host.",
 702         optional => 1,
 703     },
 704     audio0 => {
 705         type => 'string',
 706         format => $audio_fmt,
 707         description => "Configure a audio device, useful in combination with QXL/Spice.",
 708         optional => 1
 709     },
 710     spice_enhancements => {
 711         type => 'string',
 712         format => $spice_enhancements_fmt,
 713         description => "Configure additional enhancements for SPICE.",
 714         optional => 1
 715     },
 716     tags => {
 717         type => 'string', format => 'pve-tag-list',
 718         description => 'Tags of the VM. This is only meta information.',
 719         optional => 1,
 720     },
 721     rng0 => {
 722         type => 'string',
 723         format => $rng_fmt,
 724         description => "Configure a VirtIO-based Random Number Generator.",
 725         optional => 1,
 726     },
 727     meta => {
 728         type => 'string',
 729         format => $meta_info_fmt,
 730         description => "Some (read-only) meta-information about this guest.",
 731         optional => 1,
 732     },
 733     affinity => {
 734         type => 'string', format => 'pve-cpuset',
 735         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 736         optional => 1,
 737     },
 738 };
 739
 740 my $cicustom_fmt = {
 741     meta => {
 742         type => 'string',
 743         optional => 1,
 744         description => 'Specify a custom file containing all meta data passed to the VM via"
 745             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 746         format => 'pve-volume-id',
 747         format_description => 'volume',
 748     },
 749     network => {
 750         type => 'string',
 751         optional => 1,
 752         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 753         format => 'pve-volume-id',
 754         format_description => 'volume',
 755     },
 756     user => {
 757         type => 'string',
 758         optional => 1,
 759         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 760         format => 'pve-volume-id',
 761         format_description => 'volume',
 762     },
 763     vendor => {
 764         type => 'string',
 765         optional => 1,
 766         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 767         format => 'pve-volume-id',
 768         format_description => 'volume',
 769     },
 770 };
 771 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 772
 773 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 774 my $confdesc_cloudinit = {
 775     citype => {
 776         optional => 1,
 777         type => 'string',
 778         description => 'Specifies the cloud-init configuration format. The default depends on the'
 779             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 780             .' and `configdrive2` for windows.',
 781         enum => ['configdrive2', 'nocloud', 'opennebula'],
 782     },
 783     ciuser => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: User name to change ssh keys and password for instead of the"
 787             ." image's configured default user.",
 788     },
 789     cipassword => {
 790         optional => 1,
 791         type => 'string',
 792         description => 'cloud-init: Password to assign the user. Using this is generally not'
 793             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 794             .' support hashed passwords.',
 795     },
 796     ciupgrade => {
 797         optional => 1,
 798         type => 'boolean',
 799         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 800         default => 1,
 801     },
 802     cicustom => {
 803         optional => 1,
 804         type => 'string',
 805         description => 'cloud-init: Specify custom files to replace the automatically generated'
 806             .' ones at start.',
 807         format => 'pve-qm-cicustom',
 808     },
 809     searchdomain => {
 810         optional => 1,
 811         type => 'string',
 812         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 813             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 814             .' are set.',
 815     },
 816     nameserver => {
 817         optional => 1,
 818         type => 'string', format => 'address-list',
 819         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 820             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 821             .' are set.',
 822     },
 823     sshkeys => {
 824         optional => 1,
 825         type => 'string',
 826         format => 'urlencoded',
 827         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 828     },
 829 };
 830
 831 # what about other qemu settings ?
 832 #cpu => 'string',
 833 #machine => 'string',
 834 #fda => 'file',
 835 #fdb => 'file',
 836 #mtdblock => 'file',
 837 #sd => 'file',
 838 #pflash => 'file',
 839 #snapshot => 'bool',
 840 #bootp => 'file',
 841 ##tftp => 'dir',
 842 ##smb => 'dir',
 843 #kernel => 'file',
 844 #append => 'string',
 845 #initrd => 'file',
 846 ##soundhw => 'string',
 847
 848 while (my ($k, $v) = each %$confdesc) {
 849     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 850 }
 851
 852 my $MAX_NETS = 32;
 853 my $MAX_SERIAL_PORTS = 4;
 854 my $MAX_PARALLEL_PORTS = 3;
 855
 856 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 857     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 858 }
 859
 860 my $nic_model_list = [
 861     'e1000',
 862     'e1000-82540em',
 863     'e1000-82544gc',
 864     'e1000-82545em',
 865     'e1000e',
 866     'i82551',
 867     'i82557b',
 868     'i82559er',
 869     'ne2k_isa',
 870     'ne2k_pci',
 871     'pcnet',
 872     'rtl8139',
 873     'virtio',
 874     'vmxnet3',
 875 ];
 876 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 877
 878 my $net_fmt_bridge_descr = <<__EOD__;
 879 Bridge to attach the network device to. The Proxmox VE standard bridge
 880 is called 'vmbr0'.
 881
 882 If you do not specify a bridge, we create a kvm user (NATed) network
 883 device, which provides DHCP and DNS services. The following addresses
 884 are used:
 885
 886  10.0.2.2   Gateway
 887  10.0.2.3   DNS Server
 888  10.0.2.4   SMB Server
 889
 890 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 891 __EOD__
 892
 893 my $net_fmt = {
 894     macaddr  => get_standard_option('mac-addr', {
 895         description => "MAC address. That address must be unique withing your network. This is"
 896             ." automatically generated if not specified.",
 897     }),
 898     model => {
 899         type => 'string',
 900         description => "Network Card Model. The 'virtio' model provides the best performance with"
 901             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 902             ." best to use 'e1000'.",
 903         enum => $nic_model_list,
 904         default_key => 1,
 905     },
 906     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 907     bridge => get_standard_option('pve-bridge-id', {
 908         description => $net_fmt_bridge_descr,
 909         optional => 1,
 910     }),
 911     queues => {
 912         type => 'integer',
 913         minimum => 0, maximum => 64,
 914         description => 'Number of packet queues to be used on the device.',
 915         optional => 1,
 916     },
 917     rate => {
 918         type => 'number',
 919         minimum => 0,
 920         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 921         optional => 1,
 922     },
 923     tag => {
 924         type => 'integer',
 925         minimum => 1, maximum => 4094,
 926         description => 'VLAN tag to apply to packets on this interface.',
 927         optional => 1,
 928     },
 929     trunks => {
 930         type => 'string',
 931         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 932         description => 'VLAN trunks to pass through this interface.',
 933         format_description => 'vlanid[;vlanid...]',
 934         optional => 1,
 935     },
 936     firewall => {
 937         type => 'boolean',
 938         description => 'Whether this interface should be protected by the firewall.',
 939         optional => 1,
 940     },
 941     link_down => {
 942         type => 'boolean',
 943         description => 'Whether this interface should be disconnected (like pulling the plug).',
 944         optional => 1,
 945     },
 946     mtu => {
 947         type => 'integer',
 948         minimum => 1, maximum => 65520,
 949         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 950         optional => 1,
 951     },
 952 };
 953
 954 my $netdesc = {
 955     optional => 1,
 956     type => 'string', format => $net_fmt,
 957     description => "Specify network devices.",
 958 };
 959
 960 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 961
 962 my $ipconfig_fmt = {
 963     ip => {
 964         type => 'string',
 965         format => 'pve-ipv4-config',
 966         format_description => 'IPv4Format/CIDR',
 967         description => 'IPv4 address in CIDR format.',
 968         optional => 1,
 969         default => 'dhcp',
 970     },
 971     gw => {
 972         type => 'string',
 973         format => 'ipv4',
 974         format_description => 'GatewayIPv4',
 975         description => 'Default gateway for IPv4 traffic.',
 976         optional => 1,
 977         requires => 'ip',
 978     },
 979     ip6 => {
 980         type => 'string',
 981         format => 'pve-ipv6-config',
 982         format_description => 'IPv6Format/CIDR',
 983         description => 'IPv6 address in CIDR format.',
 984         optional => 1,
 985         default => 'dhcp',
 986     },
 987     gw6 => {
 988         type => 'string',
 989         format => 'ipv6',
 990         format_description => 'GatewayIPv6',
 991         description => 'Default gateway for IPv6 traffic.',
 992         optional => 1,
 993         requires => 'ip6',
 994     },
 995 };
 996 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 997 my $ipconfigdesc = {
 998     optional => 1,
 999     type => 'string', format => 'pve-qm-ipconfig',
1000     description => <<'EODESCR',
1001 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1002
1003 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1004
1005 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1006 gateway should be provided.
1007 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1008 cloud-init 19.4 or newer.
1009
1010 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1011 dhcp on IPv4.
1012 EODESCR
1013 };
1014 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1015
1016 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1017     $confdesc->{"net$i"} = $netdesc;
1018     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1019 }
1020
1021 foreach my $key (keys %$confdesc_cloudinit) {
1022     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1023 }
1024
1025 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1026 sub pve_verify_cpuset {
1027     my ($set_text, $noerr) = @_;
1028
1029     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1030
1031     if ($@) {
1032         return if $noerr;
1033         die "unable to parse cpuset option\n";
1034     }
1035
1036     return PVE::CpuSet->new($members)->short_string();
1037 }
1038
1039 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1040 sub verify_volume_id_or_qm_path {
1041     my ($volid, $noerr) = @_;
1042
1043     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1044
1045     return verify_volume_id_or_absolute_path($volid, $noerr);
1046 }
1047
1048 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1049 sub verify_volume_id_or_absolute_path {
1050     my ($volid, $noerr) = @_;
1051
1052     return $volid if $volid =~ m|^/|;
1053
1054     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1055     if ($@) {
1056         return if $noerr;
1057         die $@;
1058     }
1059     return $volid;
1060 }
1061
1062 my $serialdesc = {
1063         optional => 1,
1064         type => 'string',
1065         pattern => '(/dev/.+|socket)',
1066         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1067         verbose_description =>  <<EODESCR,
1068 Create a serial device inside the VM (n is 0 to 3), and pass through a
1069 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1070 host side (use 'qm terminal' to open a terminal connection).
1071
1072 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1073 use with special care.
1074
1075 CAUTION: Experimental! User reported problems with this option.
1076 EODESCR
1077 };
1078
1079 my $paralleldesc= {
1080         optional => 1,
1081         type => 'string',
1082         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1083         description =>  "Map host parallel devices (n is 0 to 2).",
1084         verbose_description =>  <<EODESCR,
1085 Map host parallel devices (n is 0 to 2).
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1095     $confdesc->{"parallel$i"} = $paralleldesc;
1096 }
1097
1098 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1099     $confdesc->{"serial$i"} = $serialdesc;
1100 }
1101
1102 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1103     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1104 }
1105
1106 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1107     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1108 }
1109
1110 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1111     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1112 }
1113
1114 my $boot_fmt = {
1115     legacy => {
1116         optional => 1,
1117         default_key => 1,
1118         type => 'string',
1119         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1120                      . " Deprecated, use 'order=' instead.",
1121         pattern => '[acdn]{1,4}',
1122         format_description => "[acdn]{1,4}",
1123
1124         # note: this is also the fallback if boot: is not given at all
1125         default => 'cdn',
1126     },
1127     order => {
1128         optional => 1,
1129         type => 'string',
1130         format => 'pve-qm-bootdev-list',
1131         format_description => "device[;device...]",
1132         description => <<EODESC,
1133 The guest will attempt to boot from devices in the order they appear here.
1134
1135 Disks, optical drives and passed-through storage USB devices will be directly
1136 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1137 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1138
1139 Note that only devices in this list will be marked as bootable and thus loaded
1140 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1141 (e.g. software-raid), you need to specify all of them here.
1142
1143 Overrides the deprecated 'legacy=[acdn]*' value when given.
1144 EODESC
1145     },
1146 };
1147 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1148
1149 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1150 sub verify_bootdev {
1151     my ($dev, $noerr) = @_;
1152
1153     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1154     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1155
1156     my $check = sub {
1157         my ($base) = @_;
1158         return 0 if $dev !~ m/^$base\d+$/;
1159         return 0 if !$confdesc->{$dev};
1160         return 1;
1161     };
1162
1163     return $dev if $check->("net");
1164     return $dev if $check->("usb");
1165     return $dev if $check->("hostpci");
1166
1167     return if $noerr;
1168     die "invalid boot device '$dev'\n";
1169 }
1170
1171 sub print_bootorder {
1172     my ($devs) = @_;
1173     return "" if !@$devs;
1174     my $data = { order => join(';', @$devs) };
1175     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1176 }
1177
1178 my $kvm_api_version = 0;
1179
1180 sub kvm_version {
1181     return $kvm_api_version if $kvm_api_version;
1182
1183     open my $fh, '<', '/dev/kvm' or return;
1184
1185     # 0xae00 => KVM_GET_API_VERSION
1186     $kvm_api_version = ioctl($fh, 0xae00, 0);
1187     close($fh);
1188
1189     return $kvm_api_version;
1190 }
1191
1192 my $kvm_user_version = {};
1193 my $kvm_mtime = {};
1194
1195 sub kvm_user_version {
1196     my ($binary) = @_;
1197
1198     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1199     my $st = stat($binary);
1200
1201     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1202     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1203         $cachedmtime == $st->mtime;
1204
1205     $kvm_user_version->{$binary} = 'unknown';
1206     $kvm_mtime->{$binary} = $st->mtime;
1207
1208     my $code = sub {
1209         my $line = shift;
1210         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1211             $kvm_user_version->{$binary} = $2;
1212         }
1213     };
1214
1215     eval { run_command([$binary, '--version'], outfunc => $code); };
1216     warn $@ if $@;
1217
1218     return $kvm_user_version->{$binary};
1219
1220 }
1221 my sub extract_version {
1222     my ($machine_type, $version) = @_;
1223     $version = kvm_user_version() if !defined($version);
1224     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1225 }
1226
1227 sub kernel_has_vhost_net {
1228     return -c '/dev/vhost-net';
1229 }
1230
1231 sub option_exists {
1232     my $key = shift;
1233     return defined($confdesc->{$key});
1234 }
1235
1236 my $cdrom_path;
1237 sub get_cdrom_path {
1238
1239     return $cdrom_path if defined($cdrom_path);
1240
1241     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1242
1243     if (!defined($cdrom_path)) {
1244         log_warn("no physical CD-ROM available, ignoring");
1245         $cdrom_path = '';
1246     }
1247
1248     return $cdrom_path;
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269      if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1314             if !$vtype;
1315         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1316         verify_media_type($opt, $vtype, $drive->{media});
1317         $drive->{file} = $volid;
1318     }
1319
1320     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1321 }
1322
1323 sub parse_hotplug_features {
1324     my ($data) = @_;
1325
1326     my $res = {};
1327
1328     return $res if $data eq '0';
1329
1330     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1331
1332     foreach my $feature (PVE::Tools::split_list($data)) {
1333         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1334             $res->{$1} = 1;
1335         } else {
1336             die "invalid hotplug feature '$feature'\n";
1337         }
1338     }
1339     return $res;
1340 }
1341
1342 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1343 sub pve_verify_hotplug_features {
1344     my ($value, $noerr) = @_;
1345
1346     return $value if parse_hotplug_features($value);
1347
1348     return if $noerr;
1349
1350     die "unable to parse hotplug option\n";
1351 }
1352
1353 sub assert_clipboard_config {
1354     my ($vga) = @_;
1355
1356     my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1357
1358     if (
1359         $vga->{'clipboard'}
1360         && $vga->{'clipboard'} eq 'vnc'
1361         && $vga->{type}
1362         && $vga->{type} !~ $clipboard_regex
1363     ) {
1364         die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1365     }
1366 }
1367
1368 sub print_tabletdevice_full {
1369     my ($conf, $arch) = @_;
1370
1371     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1372
1373     # we use uhci for old VMs because tablet driver was buggy in older qemu
1374     my $usbbus;
1375     if ($q35 || $arch eq 'aarch64') {
1376         $usbbus = 'ehci';
1377     } else {
1378         $usbbus = 'uhci';
1379     }
1380
1381     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1382 }
1383
1384 sub print_keyboarddevice_full {
1385     my ($conf, $arch) = @_;
1386
1387     return if $arch ne 'aarch64';
1388
1389     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1390 }
1391
1392 my sub get_drive_id {
1393     my ($drive) = @_;
1394     return "$drive->{interface}$drive->{index}";
1395 }
1396
1397 sub print_drivedevice_full {
1398     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1399
1400     my $device = '';
1401     my $maxdev = 0;
1402
1403     my $drive_id = get_drive_id($drive);
1404     if ($drive->{interface} eq 'virtio') {
1405         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1406         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1407         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1408     } elsif ($drive->{interface} eq 'scsi') {
1409
1410         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1411         my $unit = $drive->{index} % $maxdev;
1412
1413         my $machine_version = extract_version($machine_type, kvm_user_version());
1414         my $devicetype  = PVE::QemuServer::Drive::get_scsi_devicetype(
1415             $drive, $storecfg, $machine_version);
1416
1417         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1418             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1419         } else {
1420             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1421                 .",lun=$drive->{index}";
1422         }
1423         $device .= ",drive=drive-$drive_id,id=$drive_id";
1424
1425         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1426             $device .= ",rotation_rate=1";
1427         }
1428         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1429
1430         # only scsi-hd and scsi-cd support passing vendor and product information
1431         if ($devicetype eq 'hd' || $devicetype eq 'cd') {
1432             if (my $vendor = $drive->{vendor}) {
1433                 $device .= ",vendor=$vendor";
1434             }
1435             if (my $product = $drive->{product}) {
1436                 $device .= ",product=$product";
1437             }
1438         }
1439
1440     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1441         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1442         my $controller = int($drive->{index} / $maxdev);
1443         my $unit = $drive->{index} % $maxdev;
1444
1445         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1446         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1447         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1448         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1449         # were before. Move odd ones up by 2 where they don't clash.
1450         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1451             $controller += 2 * ($unit % 2);
1452             $unit = 0;
1453         }
1454
1455         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1456
1457         $device = "ide-$devicetype";
1458         if ($drive->{interface} eq 'ide') {
1459             $device .= ",bus=ide.$controller,unit=$unit";
1460         } else {
1461             $device .= ",bus=ahci$controller.$unit";
1462         }
1463         $device .= ",drive=drive-$drive_id,id=$drive_id";
1464
1465         if ($devicetype eq 'hd') {
1466             if (my $model = $drive->{model}) {
1467                 $model = URI::Escape::uri_unescape($model);
1468                 $device .= ",model=$model";
1469             }
1470             if ($drive->{ssd}) {
1471                 $device .= ",rotation_rate=1";
1472             }
1473         }
1474         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1475     } elsif ($drive->{interface} eq 'usb') {
1476         die "implement me";
1477         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1478     } else {
1479         die "unsupported interface type";
1480     }
1481
1482     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1483
1484     if (my $serial = $drive->{serial}) {
1485         $serial = URI::Escape::uri_unescape($serial);
1486         $device .= ",serial=$serial";
1487     }
1488
1489
1490     return $device;
1491 }
1492
1493 sub get_initiator_name {
1494     my $initiator;
1495
1496     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1497     while (defined(my $line = <$fh>)) {
1498         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1499         $initiator = $1;
1500         last;
1501     }
1502     $fh->close();
1503
1504     return $initiator;
1505 }
1506
1507 my sub storage_allows_io_uring_default {
1508     my ($scfg, $cache_direct) = @_;
1509
1510     # io_uring with cache mode writeback or writethrough on krbd will hang...
1511     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1512
1513     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1514     # sometimes, just plain disable...
1515     return if $scfg && $scfg->{type} eq 'lvm';
1516
1517     # io_uring causes problems when used with CIFS since kernel 5.15
1518     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1519     return if $scfg && $scfg->{type} eq 'cifs';
1520
1521     return 1;
1522 }
1523
1524 my sub drive_uses_cache_direct {
1525     my ($drive, $scfg) = @_;
1526
1527     my $cache_direct = 0;
1528
1529     if (my $cache = $drive->{cache}) {
1530         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1531     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1532         $cache_direct = 1;
1533     }
1534
1535     return $cache_direct;
1536 }
1537
1538 sub print_drive_commandline_full {
1539     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1540
1541     my $path;
1542     my $volid = $drive->{file};
1543     my $format = $drive->{format};
1544     my $drive_id = get_drive_id($drive);
1545
1546     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1547     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1548
1549     if (drive_is_cdrom($drive)) {
1550         $path = get_iso_path($storecfg, $vmid, $volid);
1551         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1552     } else {
1553         if ($storeid) {
1554             $path = PVE::Storage::path($storecfg, $volid);
1555             $format //= qemu_img_format($scfg, $volname);
1556         } else {
1557             $path = $volid;
1558             $format //= "raw";
1559         }
1560    }
1561
1562    my $is_rbd = $path =~ m/^rbd:/;
1563
1564     my $opts = '';
1565     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1566     foreach my $o (@qemu_drive_options) {
1567         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1568     }
1569
1570     # snapshot only accepts on|off
1571     if (defined($drive->{snapshot})) {
1572         my $v = $drive->{snapshot} ? 'on' : 'off';
1573         $opts .= ",snapshot=$v";
1574     }
1575
1576     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1577         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1578     }
1579
1580     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1581         my ($dir, $qmpname) = @$type;
1582         if (my $v = $drive->{"mbps$dir"}) {
1583             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1584         }
1585         if (my $v = $drive->{"mbps${dir}_max"}) {
1586             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1587         }
1588         if (my $v = $drive->{"bps${dir}_max_length"}) {
1589             $opts .= ",throttling.bps$qmpname-max-length=$v";
1590         }
1591         if (my $v = $drive->{"iops${dir}"}) {
1592             $opts .= ",throttling.iops$qmpname=$v";
1593         }
1594         if (my $v = $drive->{"iops${dir}_max"}) {
1595             $opts .= ",throttling.iops$qmpname-max=$v";
1596         }
1597         if (my $v = $drive->{"iops${dir}_max_length"}) {
1598             $opts .= ",throttling.iops$qmpname-max-length=$v";
1599         }
1600     }
1601
1602     if ($pbs_name) {
1603         $format = "rbd" if $is_rbd;
1604         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1605             if !$format;
1606         $opts .= ",format=alloc-track,file.driver=$format";
1607     } elsif ($format) {
1608         $opts .= ",format=$format";
1609     }
1610
1611     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1612
1613     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1614
1615     if (!$drive->{aio}) {
1616         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1617             # io_uring supports all cache modes
1618             $opts .= ",aio=io_uring";
1619         } else {
1620             # aio native works only with O_DIRECT
1621             if($cache_direct) {
1622                 $opts .= ",aio=native";
1623             } else {
1624                 $opts .= ",aio=threads";
1625             }
1626         }
1627     }
1628
1629     if (!drive_is_cdrom($drive)) {
1630         my $detectzeroes;
1631         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1632             $detectzeroes = 'off';
1633         } elsif ($drive->{discard}) {
1634             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1635         } else {
1636             # This used to be our default with discard not being specified:
1637             $detectzeroes = 'on';
1638         }
1639
1640         # note: 'detect-zeroes' works per blockdev and we want it to persist
1641         # after the alloc-track is removed, so put it on 'file' directly
1642         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1643         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1644     }
1645
1646     if ($pbs_name) {
1647         $opts .= ",backing=$pbs_name";
1648         $opts .= ",auto-remove=on";
1649     }
1650
1651     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1652     my $file_param = "file";
1653     if ($pbs_name) {
1654         # non-rbd drivers require the underlying file to be a seperate block
1655         # node, so add a second .file indirection
1656         $file_param .= ".file" if !$is_rbd;
1657         $file_param .= ".filename";
1658     }
1659     my $pathinfo = $path ? "$file_param=$path," : '';
1660
1661     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1662 }
1663
1664 sub print_pbs_blockdev {
1665     my ($pbs_conf, $pbs_name) = @_;
1666     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1667     $blockdev .= ",repository=$pbs_conf->{repository}";
1668     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1669     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1670     $blockdev .= ",archive=$pbs_conf->{archive}";
1671     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1672     return $blockdev;
1673 }
1674
1675 sub print_netdevice_full {
1676     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1677
1678     my $device = $net->{model};
1679     if ($net->{model} eq 'virtio') {
1680          $device = 'virtio-net-pci';
1681      };
1682
1683     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1684     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1685     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1686         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1687         # and out of each queue plus one config interrupt and control vector queue
1688         my $vectors = $net->{queues} * 2 + 2;
1689         $tmpstr .= ",vectors=$vectors,mq=on";
1690         if (min_version($machine_version, 7, 1)) {
1691             $tmpstr .= ",packed=on";
1692         }
1693     }
1694
1695     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1696         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1697     }
1698
1699     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1700
1701     if (my $mtu = $net->{mtu}) {
1702         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1703             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1704             if ($mtu == 1) {
1705                  $mtu = $bridge_mtu;
1706             } elsif ($mtu < 576) {
1707                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1708             } elsif ($mtu > $bridge_mtu) {
1709                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1710             }
1711             $tmpstr .= ",host_mtu=$mtu";
1712         } else {
1713             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1714         }
1715     }
1716
1717     if ($use_old_bios_files) {
1718         my $romfile;
1719         if ($device eq 'virtio-net-pci') {
1720             $romfile = 'pxe-virtio.rom';
1721         } elsif ($device eq 'e1000') {
1722             $romfile = 'pxe-e1000.rom';
1723         } elsif ($device eq 'e1000e') {
1724             $romfile = 'pxe-e1000e.rom';
1725         } elsif ($device eq 'ne2k') {
1726             $romfile = 'pxe-ne2k_pci.rom';
1727         } elsif ($device eq 'pcnet') {
1728             $romfile = 'pxe-pcnet.rom';
1729         } elsif ($device eq 'rtl8139') {
1730             $romfile = 'pxe-rtl8139.rom';
1731         }
1732         $tmpstr .= ",romfile=$romfile" if $romfile;
1733     }
1734
1735     return $tmpstr;
1736 }
1737
1738 sub print_netdev_full {
1739     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1740
1741     my $i = '';
1742     if ($netid =~ m/^net(\d+)$/) {
1743         $i = int($1);
1744     }
1745
1746     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1747
1748     my $ifname = "tap${vmid}i$i";
1749
1750     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1751     die "interface name '$ifname' is too long (max 15 character)\n"
1752         if length($ifname) >= 16;
1753
1754     my $vhostparam = '';
1755     if (is_native($arch)) {
1756         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1757     }
1758
1759     my $vmname = $conf->{name} || "vm$vmid";
1760
1761     my $netdev = "";
1762     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1763
1764     if ($net->{bridge}) {
1765         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1766             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1767     } else {
1768         $netdev = "type=user,id=$netid,hostname=$vmname";
1769     }
1770
1771     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1772
1773     return $netdev;
1774 }
1775
1776 my $vga_map = {
1777     'cirrus' => 'cirrus-vga',
1778     'std' => 'VGA',
1779     'vmware' => 'vmware-svga',
1780     'virtio' => 'virtio-vga',
1781     'virtio-gl' => 'virtio-vga-gl',
1782 };
1783
1784 sub print_vga_device {
1785     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1786
1787     my $type = $vga_map->{$vga->{type}};
1788     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1789         $type = 'virtio-gpu';
1790     }
1791     my $vgamem_mb = $vga->{memory};
1792
1793     my $max_outputs = '';
1794     if ($qxlnum) {
1795         $type = $id ? 'qxl' : 'qxl-vga';
1796
1797         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1798             # set max outputs so linux can have up to 4 qxl displays with one device
1799             if (min_version($machine_version, 4, 1)) {
1800                 $max_outputs = ",max_outputs=4";
1801             }
1802         }
1803     }
1804
1805     die "no devicetype for $vga->{type}\n" if !$type;
1806
1807     my $memory = "";
1808     if ($vgamem_mb) {
1809         if ($vga->{type} =~ /^virtio/) {
1810             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1811             $memory = ",max_hostmem=$bytes";
1812         } elsif ($qxlnum) {
1813             # from https://www.spice-space.org/multiple-monitors.html
1814             $memory = ",vgamem_mb=$vga->{memory}";
1815             my $ram = $vgamem_mb * 4;
1816             my $vram = $vgamem_mb * 2;
1817             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1818         } else {
1819             $memory = ",vgamem_mb=$vga->{memory}";
1820         }
1821     } elsif ($qxlnum && $id) {
1822         $memory = ",ram_size=67108864,vram_size=33554432";
1823     }
1824
1825     my $edidoff = "";
1826     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1827         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1828     }
1829
1830     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1831     my $vgaid = "vga" . ($id // '');
1832     my $pciaddr;
1833     if ($q35 && $vgaid eq 'vga') {
1834         # the first display uses pcie.0 bus on q35 machines
1835         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1836     } else {
1837         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1838     }
1839
1840     if ($vga->{type} eq 'virtio-gl') {
1841         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1842         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1843             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1844
1845         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1846             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1847     }
1848
1849     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1850 }
1851
1852 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1853 sub parse_net {
1854     my ($data, $disable_mac_autogen) = @_;
1855
1856     my $res = eval { parse_property_string($net_fmt, $data) };
1857     if ($@) {
1858         warn $@;
1859         return;
1860     }
1861     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1862         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1863         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1864     }
1865     return $res;
1866 }
1867
1868 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1869 sub parse_ipconfig {
1870     my ($data) = @_;
1871
1872     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1873     if ($@) {
1874         warn $@;
1875         return;
1876     }
1877
1878     if ($res->{gw} && !$res->{ip}) {
1879         warn 'gateway specified without specifying an IP address';
1880         return;
1881     }
1882     if ($res->{gw6} && !$res->{ip6}) {
1883         warn 'IPv6 gateway specified without specifying an IPv6 address';
1884         return;
1885     }
1886     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1887         warn 'gateway specified together with DHCP';
1888         return;
1889     }
1890     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1891         # gw6 + auto/dhcp
1892         warn "IPv6 gateway specified together with $res->{ip6} address";
1893         return;
1894     }
1895
1896     if (!$res->{ip} && !$res->{ip6}) {
1897         return { ip => 'dhcp', ip6 => 'dhcp' };
1898     }
1899
1900     return $res;
1901 }
1902
1903 sub print_net {
1904     my $net = shift;
1905
1906     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1907 }
1908
1909 sub add_random_macs {
1910     my ($settings) = @_;
1911
1912     foreach my $opt (keys %$settings) {
1913         next if $opt !~ m/^net(\d+)$/;
1914         my $net = parse_net($settings->{$opt});
1915         next if !$net;
1916         $settings->{$opt} = print_net($net);
1917     }
1918 }
1919
1920 sub vm_is_volid_owner {
1921     my ($storecfg, $vmid, $volid) = @_;
1922
1923     if ($volid !~  m|^/|) {
1924         my ($path, $owner);
1925         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1926         if ($owner && ($owner == $vmid)) {
1927             return 1;
1928         }
1929     }
1930
1931     return;
1932 }
1933
1934 sub vmconfig_register_unused_drive {
1935     my ($storecfg, $vmid, $conf, $drive) = @_;
1936
1937     if (drive_is_cloudinit($drive)) {
1938         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1939         warn $@ if $@;
1940         delete $conf->{cloudinit};
1941     } elsif (!drive_is_cdrom($drive)) {
1942         my $volid = $drive->{file};
1943         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1944             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1945         }
1946     }
1947 }
1948
1949 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1950 my $smbios1_fmt = {
1951     uuid => {
1952         type => 'string',
1953         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1954         format_description => 'UUID',
1955         description => "Set SMBIOS1 UUID.",
1956         optional => 1,
1957     },
1958     version => {
1959         type => 'string',
1960         pattern => '[A-Za-z0-9+\/]+={0,2}',
1961         format_description => 'Base64 encoded string',
1962         description => "Set SMBIOS1 version.",
1963         optional => 1,
1964     },
1965     serial => {
1966         type => 'string',
1967         pattern => '[A-Za-z0-9+\/]+={0,2}',
1968         format_description => 'Base64 encoded string',
1969         description => "Set SMBIOS1 serial number.",
1970         optional => 1,
1971     },
1972     manufacturer => {
1973         type => 'string',
1974         pattern => '[A-Za-z0-9+\/]+={0,2}',
1975         format_description => 'Base64 encoded string',
1976         description => "Set SMBIOS1 manufacturer.",
1977         optional => 1,
1978     },
1979     product => {
1980         type => 'string',
1981         pattern => '[A-Za-z0-9+\/]+={0,2}',
1982         format_description => 'Base64 encoded string',
1983         description => "Set SMBIOS1 product ID.",
1984         optional => 1,
1985     },
1986     sku => {
1987         type => 'string',
1988         pattern => '[A-Za-z0-9+\/]+={0,2}',
1989         format_description => 'Base64 encoded string',
1990         description => "Set SMBIOS1 SKU string.",
1991         optional => 1,
1992     },
1993     family => {
1994         type => 'string',
1995         pattern => '[A-Za-z0-9+\/]+={0,2}',
1996         format_description => 'Base64 encoded string',
1997         description => "Set SMBIOS1 family string.",
1998         optional => 1,
1999     },
2000     base64 => {
2001         type => 'boolean',
2002         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2003         optional => 1,
2004     },
2005 };
2006
2007 sub parse_smbios1 {
2008     my ($data) = @_;
2009
2010     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2011     warn $@ if $@;
2012     return $res;
2013 }
2014
2015 sub print_smbios1 {
2016     my ($smbios1) = @_;
2017     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2018 }
2019
2020 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2021
2022 sub parse_watchdog {
2023     my ($value) = @_;
2024
2025     return if !$value;
2026
2027     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2028     warn $@ if $@;
2029     return $res;
2030 }
2031
2032 sub parse_guest_agent {
2033     my ($conf) = @_;
2034
2035     return {} if !defined($conf->{agent});
2036
2037     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2038     warn $@ if $@;
2039
2040     # if the agent is disabled ignore the other potentially set properties
2041     return {} if !$res->{enabled};
2042     return $res;
2043 }
2044
2045 sub get_qga_key {
2046     my ($conf, $key) = @_;
2047     return undef if !defined($conf->{agent});
2048
2049     my $agent = parse_guest_agent($conf);
2050     return $agent->{$key};
2051 }
2052
2053 sub parse_vga {
2054     my ($value) = @_;
2055
2056     return {} if !$value;
2057     my $res = eval { parse_property_string($vga_fmt, $value) };
2058     warn $@ if $@;
2059     return $res;
2060 }
2061
2062 sub parse_rng {
2063     my ($value) = @_;
2064
2065     return if !$value;
2066
2067     my $res = eval { parse_property_string($rng_fmt, $value) };
2068     warn $@ if $@;
2069     return $res;
2070 }
2071
2072 sub parse_meta_info {
2073     my ($value) = @_;
2074
2075     return if !$value;
2076
2077     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2078     warn $@ if $@;
2079     return $res;
2080 }
2081
2082 sub new_meta_info_string {
2083     my () = @_; # for now do not allow to override any value
2084
2085     return PVE::JSONSchema::print_property_string(
2086         {
2087             'creation-qemu' => kvm_user_version(),
2088             ctime => "". int(time()),
2089         },
2090         $meta_info_fmt
2091     );
2092 }
2093
2094 sub qemu_created_version_fixups {
2095     my ($conf, $forcemachine, $kvmver) = @_;
2096
2097     my $meta = parse_meta_info($conf->{meta}) // {};
2098     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2099
2100     # check if we need to apply some handling for VMs that always use the latest machine version but
2101     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2102     # would be required (we do not want to pin machine version for non-windows OS type)
2103     if (
2104         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2105         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2106         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2107         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2108     ) {
2109         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2110         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2111             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2112             # and thus with the predictable interface naming of systemd
2113             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2114         }
2115     }
2116     return;
2117 }
2118
2119 # add JSON properties for create and set function
2120 sub json_config_properties {
2121     my ($prop, $with_disk_alloc) = @_;
2122
2123     my $skip_json_config_opts = {
2124         parent => 1,
2125         snaptime => 1,
2126         vmstate => 1,
2127         runningmachine => 1,
2128         runningcpu => 1,
2129         meta => 1,
2130     };
2131
2132     foreach my $opt (keys %$confdesc) {
2133         next if $skip_json_config_opts->{$opt};
2134
2135         if ($with_disk_alloc && is_valid_drivename($opt)) {
2136             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2137         } else {
2138             $prop->{$opt} = $confdesc->{$opt};
2139         }
2140     }
2141
2142     return $prop;
2143 }
2144
2145 # Properties that we can read from an OVF file
2146 sub json_ovf_properties {
2147     my $prop = {};
2148
2149     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2150         $prop->{$device} = {
2151             type => 'string',
2152             format => 'pve-volume-id-or-absolute-path',
2153             description => "Disk image that gets imported to $device",
2154             optional => 1,
2155         };
2156     }
2157
2158     $prop->{cores} = {
2159         type => 'integer',
2160         description => "The number of CPU cores.",
2161         optional => 1,
2162     };
2163     $prop->{memory} = {
2164         type => 'integer',
2165         description => "Amount of RAM for the VM in MB.",
2166         optional => 1,
2167     };
2168     $prop->{name} = {
2169         type => 'string',
2170         description => "Name of the VM.",
2171         optional => 1,
2172     };
2173
2174     return $prop;
2175 }
2176
2177 # return copy of $confdesc_cloudinit to generate documentation
2178 sub cloudinit_config_properties {
2179
2180     return dclone($confdesc_cloudinit);
2181 }
2182
2183 sub cloudinit_pending_properties {
2184     my $p = {
2185         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2186         name => 1,
2187     };
2188     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2189     return $p;
2190 }
2191
2192 sub check_type {
2193     my ($key, $value) = @_;
2194
2195     die "unknown setting '$key'\n" if !$confdesc->{$key};
2196
2197     my $type = $confdesc->{$key}->{type};
2198
2199     if (!defined($value)) {
2200         die "got undefined value\n";
2201     }
2202
2203     if ($value =~ m/[\n\r]/) {
2204         die "property contains a line feed\n";
2205     }
2206
2207     if ($type eq 'boolean') {
2208         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2209         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2210         die "type check ('boolean') failed - got '$value'\n";
2211     } elsif ($type eq 'integer') {
2212         return int($1) if $value =~ m/^(\d+)$/;
2213         die "type check ('integer') failed - got '$value'\n";
2214     } elsif ($type eq 'number') {
2215         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2216         die "type check ('number') failed - got '$value'\n";
2217     } elsif ($type eq 'string') {
2218         if (my $fmt = $confdesc->{$key}->{format}) {
2219             PVE::JSONSchema::check_format($fmt, $value);
2220             return $value;
2221         }
2222         $value =~ s/^\"(.*)\"$/$1/;
2223         return $value;
2224     } else {
2225         die "internal error"
2226     }
2227 }
2228
2229 sub destroy_vm {
2230     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2231
2232     my $conf = PVE::QemuConfig->load_config($vmid);
2233
2234     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2235         PVE::QemuConfig->check_lock($conf);
2236     }
2237
2238     if ($conf->{template}) {
2239         # check if any base image is still used by a linked clone
2240         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2241                 my ($ds, $drive) = @_;
2242                 return if drive_is_cdrom($drive);
2243
2244                 my $volid = $drive->{file};
2245                 return if !$volid || $volid =~ m|^/|;
2246
2247                 die "base volume '$volid' is still in use by linked cloned\n"
2248                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2249
2250         });
2251     }
2252
2253     my $volids = {};
2254     my $remove_owned_drive = sub {
2255         my ($ds, $drive) = @_;
2256         return if drive_is_cdrom($drive, 1);
2257
2258         my $volid = $drive->{file};
2259         return if !$volid || $volid =~ m|^/|;
2260         return if $volids->{$volid};
2261
2262         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2263         return if !$path || !$owner || ($owner != $vmid);
2264
2265         $volids->{$volid} = 1;
2266         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2267         warn "Could not remove disk '$volid', check manually: $@" if $@;
2268     };
2269
2270     # only remove disks owned by this VM (referenced in the config)
2271     my $include_opts = {
2272         include_unused => 1,
2273         extra_keys => ['vmstate'],
2274     };
2275     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2276
2277     for my $snap (values %{$conf->{snapshots}}) {
2278         next if !defined($snap->{vmstate});
2279         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2280         next if !defined($drive);
2281         $remove_owned_drive->('vmstate', $drive);
2282     }
2283
2284     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2285
2286     if ($purge_unreferenced) { # also remove unreferenced disk
2287         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2288         PVE::Storage::foreach_volid($vmdisks, sub {
2289             my ($volid, $sid, $volname, $d) = @_;
2290             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2291             warn $@ if $@;
2292         });
2293     }
2294
2295     eval { delete_ifaces_ipams_ips($conf, $vmid)};
2296     warn $@ if $@;
2297
2298     if (defined $replacement_conf) {
2299         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2300     } else {
2301         PVE::QemuConfig->destroy_config($vmid);
2302     }
2303 }
2304
2305 sub parse_vm_config {
2306     my ($filename, $raw, $strict) = @_;
2307
2308     return if !defined($raw);
2309
2310     my $res = {
2311         digest => Digest::SHA::sha1_hex($raw),
2312         snapshots => {},
2313         pending => {},
2314         cloudinit => {},
2315     };
2316
2317     my $handle_error = sub {
2318         my ($msg) = @_;
2319
2320         if ($strict) {
2321             die $msg;
2322         } else {
2323             warn $msg;
2324         }
2325     };
2326
2327     $filename =~ m|/qemu-server/(\d+)\.conf$|
2328         || die "got strange filename '$filename'";
2329
2330     my $vmid = $1;
2331
2332     my $conf = $res;
2333     my $descr;
2334     my $finish_description = sub {
2335         if (defined($descr)) {
2336             $descr =~ s/\s+$//;
2337             $conf->{description} = $descr;
2338         }
2339         $descr = undef;
2340     };
2341     my $section = '';
2342
2343     my @lines = split(/\n/, $raw);
2344     foreach my $line (@lines) {
2345         next if $line =~ m/^\s*$/;
2346
2347         if ($line =~ m/^\[PENDING\]\s*$/i) {
2348             $section = 'pending';
2349             $finish_description->();
2350             $conf = $res->{$section} = {};
2351             next;
2352         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2353             $section = 'cloudinit';
2354             $finish_description->();
2355             $conf = $res->{$section} = {};
2356             next;
2357
2358         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2359             $section = $1;
2360             $finish_description->();
2361             $conf = $res->{snapshots}->{$section} = {};
2362             next;
2363         }
2364
2365         if ($line =~ m/^\#(.*)$/) {
2366             $descr = '' if !defined($descr);
2367             $descr .= PVE::Tools::decode_text($1) . "\n";
2368             next;
2369         }
2370
2371         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2372             $descr = '' if !defined($descr);
2373             $descr .= PVE::Tools::decode_text($2);
2374         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2375             $conf->{snapstate} = $1;
2376         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2377             my $key = $1;
2378             my $value = $2;
2379             $conf->{$key} = $value;
2380         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2381             my $value = $1;
2382             if ($section eq 'pending') {
2383                 $conf->{delete} = $value; # we parse this later
2384             } else {
2385                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2386             }
2387         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2388             my $key = $1;
2389             my $value = $2;
2390             if ($section eq 'cloudinit') {
2391                 # ignore validation only used for informative purpose
2392                 $conf->{$key} = $value;
2393                 next;
2394             }
2395             eval { $value = check_type($key, $value); };
2396             if ($@) {
2397                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2398             } else {
2399                 $key = 'ide2' if $key eq 'cdrom';
2400                 my $fmt = $confdesc->{$key}->{format};
2401                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2402                     my $v = parse_drive($key, $value);
2403                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2404                         $v->{file} = $volid;
2405                         $value = print_drive($v);
2406                     } else {
2407                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2408                         next;
2409                     }
2410                 }
2411
2412                 $conf->{$key} = $value;
2413             }
2414         } else {
2415             $handle_error->("vm $vmid - unable to parse config: $line\n");
2416         }
2417     }
2418
2419     $finish_description->();
2420     delete $res->{snapstate}; # just to be sure
2421
2422     return $res;
2423 }
2424
2425 sub write_vm_config {
2426     my ($filename, $conf) = @_;
2427
2428     delete $conf->{snapstate}; # just to be sure
2429
2430     if ($conf->{cdrom}) {
2431         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2432         $conf->{ide2} = $conf->{cdrom};
2433         delete $conf->{cdrom};
2434     }
2435
2436     # we do not use 'smp' any longer
2437     if ($conf->{sockets}) {
2438         delete $conf->{smp};
2439     } elsif ($conf->{smp}) {
2440         $conf->{sockets} = $conf->{smp};
2441         delete $conf->{cores};
2442         delete $conf->{smp};
2443     }
2444
2445     my $used_volids = {};
2446
2447     my $cleanup_config = sub {
2448         my ($cref, $pending, $snapname) = @_;
2449
2450         foreach my $key (keys %$cref) {
2451             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2452                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2453             my $value = $cref->{$key};
2454             if ($key eq 'delete') {
2455                 die "propertry 'delete' is only allowed in [PENDING]\n"
2456                     if !$pending;
2457                 # fixme: check syntax?
2458                 next;
2459             }
2460             eval { $value = check_type($key, $value); };
2461             die "unable to parse value of '$key' - $@" if $@;
2462
2463             $cref->{$key} = $value;
2464
2465             if (!$snapname && is_valid_drivename($key)) {
2466                 my $drive = parse_drive($key, $value);
2467                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2468             }
2469         }
2470     };
2471
2472     &$cleanup_config($conf);
2473
2474     &$cleanup_config($conf->{pending}, 1);
2475
2476     foreach my $snapname (keys %{$conf->{snapshots}}) {
2477         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2478         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2479     }
2480
2481     # remove 'unusedX' settings if we re-add a volume
2482     foreach my $key (keys %$conf) {
2483         my $value = $conf->{$key};
2484         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2485             delete $conf->{$key};
2486         }
2487     }
2488
2489     my $generate_raw_config = sub {
2490         my ($conf, $pending) = @_;
2491
2492         my $raw = '';
2493
2494         # add description as comment to top of file
2495         if (defined(my $descr = $conf->{description})) {
2496             if ($descr) {
2497                 foreach my $cl (split(/\n/, $descr)) {
2498                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2499                 }
2500             } else {
2501                 $raw .= "#\n" if $pending;
2502             }
2503         }
2504
2505         foreach my $key (sort keys %$conf) {
2506             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2507             $raw .= "$key: $conf->{$key}\n";
2508         }
2509         return $raw;
2510     };
2511
2512     my $raw = &$generate_raw_config($conf);
2513
2514     if (scalar(keys %{$conf->{pending}})){
2515         $raw .= "\n[PENDING]\n";
2516         $raw .= &$generate_raw_config($conf->{pending}, 1);
2517     }
2518
2519     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2520         $raw .= "\n[special:cloudinit]\n";
2521         $raw .= &$generate_raw_config($conf->{cloudinit});
2522     }
2523
2524     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2525         $raw .= "\n[$snapname]\n";
2526         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2527     }
2528
2529     return $raw;
2530 }
2531
2532 sub load_defaults {
2533
2534     my $res = {};
2535
2536     # we use static defaults from our JSON schema configuration
2537     foreach my $key (keys %$confdesc) {
2538         if (defined(my $default = $confdesc->{$key}->{default})) {
2539             $res->{$key} = $default;
2540         }
2541     }
2542
2543     return $res;
2544 }
2545
2546 sub config_list {
2547     my $vmlist = PVE::Cluster::get_vmlist();
2548     my $res = {};
2549     return $res if !$vmlist || !$vmlist->{ids};
2550     my $ids = $vmlist->{ids};
2551     my $nodename = nodename();
2552
2553     foreach my $vmid (keys %$ids) {
2554         my $d = $ids->{$vmid};
2555         next if !$d->{node} || $d->{node} ne $nodename;
2556         next if !$d->{type} || $d->{type} ne 'qemu';
2557         $res->{$vmid}->{exists} = 1;
2558     }
2559     return $res;
2560 }
2561
2562 # test if VM uses local resources (to prevent migration)
2563 sub check_local_resources {
2564     my ($conf, $noerr) = @_;
2565
2566     my @loc_res = ();
2567     my $mapped_res = [];
2568
2569     my $nodelist = PVE::Cluster::get_nodelist();
2570     my $pci_map = PVE::Mapping::PCI::config();
2571     my $usb_map = PVE::Mapping::USB::config();
2572
2573     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2574
2575     my $add_missing_mapping = sub {
2576         my ($type, $key, $id) = @_;
2577         for my $node (@$nodelist) {
2578             my $entry;
2579             if ($type eq 'pci') {
2580                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2581             } elsif ($type eq 'usb') {
2582                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2583             }
2584             if (!scalar($entry->@*)) {
2585                 push @{$missing_mappings_by_node->{$node}}, $key;
2586             }
2587         }
2588     };
2589
2590     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2591     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2592
2593     push @loc_res, "ivshmem" if $conf->{ivshmem};
2594
2595     foreach my $k (keys %$conf) {
2596         if ($k =~ m/^usb/) {
2597             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2598             next if $entry->{host} =~ m/^spice$/i;
2599             if ($entry->{mapping}) {
2600                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2601                 push @$mapped_res, $k;
2602             }
2603         }
2604         if ($k =~ m/^hostpci/) {
2605             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2606             if ($entry->{mapping}) {
2607                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2608                 push @$mapped_res, $k;
2609             }
2610         }
2611         # sockets are safe: they will recreated be on the target side post-migrate
2612         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2613         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2614     }
2615
2616     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2617
2618     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2619 }
2620
2621 # check if used storages are available on all nodes (use by migrate)
2622 sub check_storage_availability {
2623     my ($storecfg, $conf, $node) = @_;
2624
2625     PVE::QemuConfig->foreach_volume($conf, sub {
2626         my ($ds, $drive) = @_;
2627
2628         my $volid = $drive->{file};
2629         return if !$volid;
2630
2631         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2632         return if !$sid;
2633
2634         # check if storage is available on both nodes
2635         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2636         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2637
2638         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2639
2640         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2641             if !$scfg->{content}->{$vtype};
2642    });
2643 }
2644
2645 # list nodes where all VM images are available (used by has_feature API)
2646 sub shared_nodes {
2647     my ($conf, $storecfg) = @_;
2648
2649     my $nodelist = PVE::Cluster::get_nodelist();
2650     my $nodehash = { map { $_ => 1 } @$nodelist };
2651     my $nodename = nodename();
2652
2653     PVE::QemuConfig->foreach_volume($conf, sub {
2654         my ($ds, $drive) = @_;
2655
2656         my $volid = $drive->{file};
2657         return if !$volid;
2658
2659         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2660         if ($storeid) {
2661             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2662             if ($scfg->{disable}) {
2663                 $nodehash = {};
2664             } elsif (my $avail = $scfg->{nodes}) {
2665                 foreach my $node (keys %$nodehash) {
2666                     delete $nodehash->{$node} if !$avail->{$node};
2667                 }
2668             } elsif (!$scfg->{shared}) {
2669                 foreach my $node (keys %$nodehash) {
2670                     delete $nodehash->{$node} if $node ne $nodename
2671                 }
2672             }
2673         }
2674     });
2675
2676     return $nodehash
2677 }
2678
2679 sub check_local_storage_availability {
2680     my ($conf, $storecfg) = @_;
2681
2682     my $nodelist = PVE::Cluster::get_nodelist();
2683     my $nodehash = { map { $_ => {} } @$nodelist };
2684
2685     PVE::QemuConfig->foreach_volume($conf, sub {
2686         my ($ds, $drive) = @_;
2687
2688         my $volid = $drive->{file};
2689         return if !$volid;
2690
2691         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2692         if ($storeid) {
2693             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2694
2695             if ($scfg->{disable}) {
2696                 foreach my $node (keys %$nodehash) {
2697                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2698                 }
2699             } elsif (my $avail = $scfg->{nodes}) {
2700                 foreach my $node (keys %$nodehash) {
2701                     if (!$avail->{$node}) {
2702                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2703                     }
2704                 }
2705             }
2706         }
2707     });
2708
2709     foreach my $node (values %$nodehash) {
2710         if (my $unavail = $node->{unavailable_storages}) {
2711             $node->{unavailable_storages} = [ sort keys %$unavail ];
2712         }
2713     }
2714
2715     return $nodehash
2716 }
2717
2718 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2719 sub check_running {
2720     my ($vmid, $nocheck, $node) = @_;
2721
2722     # $nocheck is set when called during a migration, in which case the config
2723     # file might still or already reside on the *other* node
2724     # - because rename has already happened, and current node is source
2725     # - because rename hasn't happened yet, and current node is target
2726     # - because rename has happened, current node is target, but hasn't yet
2727     # processed it yet
2728     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2729     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2730 }
2731
2732 sub vzlist {
2733
2734     my $vzlist = config_list();
2735
2736     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2737
2738     while (defined(my $de = $fd->read)) {
2739         next if $de !~ m/^(\d+)\.pid$/;
2740         my $vmid = $1;
2741         next if !defined($vzlist->{$vmid});
2742         if (my $pid = check_running($vmid)) {
2743             $vzlist->{$vmid}->{pid} = $pid;
2744         }
2745     }
2746
2747     return $vzlist;
2748 }
2749
2750 our $vmstatus_return_properties = {
2751     vmid => get_standard_option('pve-vmid'),
2752     status => {
2753         description => "QEMU process status.",
2754         type => 'string',
2755         enum => ['stopped', 'running'],
2756     },
2757     maxmem => {
2758         description => "Maximum memory in bytes.",
2759         type => 'integer',
2760         optional => 1,
2761         renderer => 'bytes',
2762     },
2763     maxdisk => {
2764         description => "Root disk size in bytes.",
2765         type => 'integer',
2766         optional => 1,
2767         renderer => 'bytes',
2768     },
2769     name => {
2770         description => "VM name.",
2771         type => 'string',
2772         optional => 1,
2773     },
2774     qmpstatus => {
2775         description => "VM run state from the 'query-status' QMP monitor command.",
2776         type => 'string',
2777         optional => 1,
2778     },
2779     pid => {
2780         description => "PID of running qemu process.",
2781         type => 'integer',
2782         optional => 1,
2783     },
2784     uptime => {
2785         description => "Uptime.",
2786         type => 'integer',
2787         optional => 1,
2788         renderer => 'duration',
2789     },
2790     cpus => {
2791         description => "Maximum usable CPUs.",
2792         type => 'number',
2793         optional => 1,
2794     },
2795     lock => {
2796         description => "The current config lock, if any.",
2797         type => 'string',
2798         optional => 1,
2799     },
2800     tags => {
2801         description  => "The current configured tags, if any",
2802         type => 'string',
2803         optional => 1,
2804     },
2805     'running-machine' => {
2806         description => "The currently running machine type (if running).",
2807         type => 'string',
2808         optional => 1,
2809     },
2810     'running-qemu' => {
2811         description => "The currently running QEMU version (if running).",
2812         type => 'string',
2813         optional => 1,
2814     },
2815 };
2816
2817 my $last_proc_pid_stat;
2818
2819 # get VM status information
2820 # This must be fast and should not block ($full == false)
2821 # We only query KVM using QMP if $full == true (this can be slow)
2822 sub vmstatus {
2823     my ($opt_vmid, $full) = @_;
2824
2825     my $res = {};
2826
2827     my $storecfg = PVE::Storage::config();
2828
2829     my $list = vzlist();
2830     my $defaults = load_defaults();
2831
2832     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2833
2834     my $cpucount = $cpuinfo->{cpus} || 1;
2835
2836     foreach my $vmid (keys %$list) {
2837         next if $opt_vmid && ($vmid ne $opt_vmid);
2838
2839         my $conf = PVE::QemuConfig->load_config($vmid);
2840
2841         my $d = { vmid => int($vmid) };
2842         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2843
2844         # fixme: better status?
2845         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2846
2847         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2848         if (defined($size)) {
2849             $d->{disk} = 0; # no info available
2850             $d->{maxdisk} = $size;
2851         } else {
2852             $d->{disk} = 0;
2853             $d->{maxdisk} = 0;
2854         }
2855
2856         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2857             * ($conf->{cores} || $defaults->{cores});
2858         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2859         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2860
2861         $d->{name} = $conf->{name} || "VM $vmid";
2862         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2863
2864         if ($conf->{balloon}) {
2865             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2866             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2867                 : $defaults->{shares};
2868         }
2869
2870         $d->{uptime} = 0;
2871         $d->{cpu} = 0;
2872         $d->{mem} = 0;
2873
2874         $d->{netout} = 0;
2875         $d->{netin} = 0;
2876
2877         $d->{diskread} = 0;
2878         $d->{diskwrite} = 0;
2879
2880         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2881
2882         $d->{serial} = 1 if conf_has_serial($conf);
2883         $d->{lock} = $conf->{lock} if $conf->{lock};
2884         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2885
2886         $res->{$vmid} = $d;
2887     }
2888
2889     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2890     foreach my $dev (keys %$netdev) {
2891         next if $dev !~ m/^tap([1-9]\d*)i/;
2892         my $vmid = $1;
2893         my $d = $res->{$vmid};
2894         next if !$d;
2895
2896         $d->{netout} += $netdev->{$dev}->{receive};
2897         $d->{netin} += $netdev->{$dev}->{transmit};
2898
2899         if ($full) {
2900             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2901             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2902         }
2903
2904     }
2905
2906     my $ctime = gettimeofday;
2907
2908     foreach my $vmid (keys %$list) {
2909
2910         my $d = $res->{$vmid};
2911         my $pid = $d->{pid};
2912         next if !$pid;
2913
2914         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2915         next if !$pstat; # not running
2916
2917         my $used = $pstat->{utime} + $pstat->{stime};
2918
2919         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2920
2921         if ($pstat->{vsize}) {
2922             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2923         }
2924
2925         my $old = $last_proc_pid_stat->{$pid};
2926         if (!$old) {
2927             $last_proc_pid_stat->{$pid} = {
2928                 time => $ctime,
2929                 used => $used,
2930                 cpu => 0,
2931             };
2932             next;
2933         }
2934
2935         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2936
2937         if ($dtime > 1000) {
2938             my $dutime = $used -  $old->{used};
2939
2940             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2941             $last_proc_pid_stat->{$pid} = {
2942                 time => $ctime,
2943                 used => $used,
2944                 cpu => $d->{cpu},
2945             };
2946         } else {
2947             $d->{cpu} = $old->{cpu};
2948         }
2949     }
2950
2951     return $res if !$full;
2952
2953     my $qmpclient = PVE::QMPClient->new();
2954
2955     my $ballooncb = sub {
2956         my ($vmid, $resp) = @_;
2957
2958         my $info = $resp->{'return'};
2959         return if !$info->{max_mem};
2960
2961         my $d = $res->{$vmid};
2962
2963         # use memory assigned to VM
2964         $d->{maxmem} = $info->{max_mem};
2965         $d->{balloon} = $info->{actual};
2966
2967         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2968             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2969             $d->{freemem} = $info->{free_mem};
2970         }
2971
2972         $d->{ballooninfo} = $info;
2973     };
2974
2975     my $blockstatscb = sub {
2976         my ($vmid, $resp) = @_;
2977         my $data = $resp->{'return'} || [];
2978         my $totalrdbytes = 0;
2979         my $totalwrbytes = 0;
2980
2981         for my $blockstat (@$data) {
2982             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2983             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2984
2985             $blockstat->{device} =~ s/drive-//;
2986             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2987         }
2988         $res->{$vmid}->{diskread} = $totalrdbytes;
2989         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2990     };
2991
2992     my $machinecb = sub {
2993         my ($vmid, $resp) = @_;
2994         my $data = $resp->{'return'} || [];
2995
2996         $res->{$vmid}->{'running-machine'} =
2997             PVE::QemuServer::Machine::current_from_query_machines($data);
2998     };
2999
3000     my $versioncb = sub {
3001         my ($vmid, $resp) = @_;
3002         my $data = $resp->{'return'} // {};
3003         my $version = 'unknown';
3004
3005         if (my $v = $data->{qemu}) {
3006             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3007         }
3008
3009         $res->{$vmid}->{'running-qemu'} = $version;
3010     };
3011
3012     my $statuscb = sub {
3013         my ($vmid, $resp) = @_;
3014
3015         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3016         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3017         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3018         # this fails if ballon driver is not loaded, so this must be
3019         # the last commnand (following command are aborted if this fails).
3020         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3021
3022         my $status = 'unknown';
3023         if (!defined($status = $resp->{'return'}->{status})) {
3024             warn "unable to get VM status\n";
3025             return;
3026         }
3027
3028         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3029     };
3030
3031     foreach my $vmid (keys %$list) {
3032         next if $opt_vmid && ($vmid ne $opt_vmid);
3033         next if !$res->{$vmid}->{pid}; # not running
3034         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3035     }
3036
3037     $qmpclient->queue_execute(undef, 2);
3038
3039     foreach my $vmid (keys %$list) {
3040         next if $opt_vmid && ($vmid ne $opt_vmid);
3041         next if !$res->{$vmid}->{pid}; #not running
3042
3043         # we can't use the $qmpclient since it might have already aborted on
3044         # 'query-balloon', but this might also fail for older versions...
3045         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3046         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3047     }
3048
3049     foreach my $vmid (keys %$list) {
3050         next if $opt_vmid && ($vmid ne $opt_vmid);
3051         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3052     }
3053
3054     return $res;
3055 }
3056
3057 sub conf_has_serial {
3058     my ($conf) = @_;
3059
3060     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3061         if ($conf->{"serial$i"}) {
3062             return 1;
3063         }
3064     }
3065
3066     return 0;
3067 }
3068
3069 sub conf_has_audio {
3070     my ($conf, $id) = @_;
3071
3072     $id //= 0;
3073     my $audio = $conf->{"audio$id"};
3074     return if !defined($audio);
3075
3076     my $audioproperties = parse_property_string($audio_fmt, $audio);
3077     my $audiodriver = $audioproperties->{driver} // 'spice';
3078
3079     return {
3080         dev => $audioproperties->{device},
3081         dev_id => "audiodev$id",
3082         backend => $audiodriver,
3083         backend_id => "$audiodriver-backend${id}",
3084     };
3085 }
3086
3087 sub audio_devs {
3088     my ($audio, $audiopciaddr, $machine_version) = @_;
3089
3090     my $devs = [];
3091
3092     my $id = $audio->{dev_id};
3093     my $audiodev = "";
3094     if (min_version($machine_version, 4, 2)) {
3095         $audiodev = ",audiodev=$audio->{backend_id}";
3096     }
3097
3098     if ($audio->{dev} eq 'AC97') {
3099         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3100     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3101         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3102         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3103         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3104     } else {
3105         die "unkown audio device '$audio->{dev}', implement me!";
3106     }
3107
3108     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3109
3110     return $devs;
3111 }
3112
3113 sub get_tpm_paths {
3114     my ($vmid) = @_;
3115     return {
3116         socket => "/var/run/qemu-server/$vmid.swtpm",
3117         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3118     };
3119 }
3120
3121 sub add_tpm_device {
3122     my ($vmid, $devices, $conf) = @_;
3123
3124     return if !$conf->{tpmstate0};
3125
3126     my $paths = get_tpm_paths($vmid);
3127
3128     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3129     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3130     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3131 }
3132
3133 sub start_swtpm {
3134     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3135
3136     return if !$tpmdrive;
3137
3138     my $state;
3139     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3140     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3141     if ($storeid) {
3142         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3143     } else {
3144         $state = $tpm->{file};
3145     }
3146
3147     my $paths = get_tpm_paths($vmid);
3148
3149     # during migration, we will get state from remote
3150     #
3151     if (!$migration) {
3152         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3153         my $setup_cmd = [
3154             "swtpm_setup",
3155             "--tpmstate",
3156             "file://$state",
3157             "--createek",
3158             "--create-ek-cert",
3159             "--create-platform-cert",
3160             "--lock-nvram",
3161             "--config",
3162             "/etc/swtpm_setup.conf", # do not use XDG configs
3163             "--runas",
3164             "0", # force creation as root, error if not possible
3165             "--not-overwrite", # ignore existing state, do not modify
3166         ];
3167
3168         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3169         # TPM 2.0 supports ECC crypto, use if possible
3170         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3171
3172         run_command($setup_cmd, outfunc => sub {
3173             print "swtpm_setup: $1\n";
3174         });
3175     }
3176
3177     # Used to distinguish different invocations in the log.
3178     my $log_prefix = "[id=" . int(time()) . "] ";
3179
3180     my $emulator_cmd = [
3181         "swtpm",
3182         "socket",
3183         "--tpmstate",
3184         "backend-uri=file://$state,mode=0600",
3185         "--ctrl",
3186         "type=unixio,path=$paths->{socket},mode=0600",
3187         "--pid",
3188         "file=$paths->{pid}",
3189         "--terminate", # terminate on QEMU disconnect
3190         "--daemon",
3191         "--log",
3192         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3193     ];
3194     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3195     run_command($emulator_cmd, outfunc => sub { print $1; });
3196
3197     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3198     while (! -e $paths->{pid}) {
3199         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3200         usleep(50_000);
3201     }
3202
3203     # return untainted PID of swtpm daemon so it can be killed on error
3204     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3205     return $1;
3206 }
3207
3208 sub vga_conf_has_spice {
3209     my ($vga) = @_;
3210
3211     my $vgaconf = parse_vga($vga);
3212     my $vgatype = $vgaconf->{type};
3213     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3214
3215     return $1 || 1;
3216 }
3217
3218 sub is_native($) {
3219     my ($arch) = @_;
3220     return get_host_arch() eq $arch;
3221 }
3222
3223 sub get_vm_arch {
3224     my ($conf) = @_;
3225     return $conf->{arch} // get_host_arch();
3226 }
3227
3228 my $default_machines = {
3229     x86_64 => 'pc',
3230     aarch64 => 'virt',
3231 };
3232
3233 sub get_installed_machine_version {
3234     my ($kvmversion) = @_;
3235     $kvmversion = kvm_user_version() if !defined($kvmversion);
3236     $kvmversion =~ m/^(\d+\.\d+)/;
3237     return $1;
3238 }
3239
3240 sub windows_get_pinned_machine_version {
3241     my ($machine, $base_version, $kvmversion) = @_;
3242
3243     my $pin_version = $base_version;
3244     if (!defined($base_version) ||
3245         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3246     ) {
3247         $pin_version = get_installed_machine_version($kvmversion);
3248     }
3249     if (!$machine || $machine eq 'pc') {
3250         $machine = "pc-i440fx-$pin_version";
3251     } elsif ($machine eq 'q35') {
3252         $machine = "pc-q35-$pin_version";
3253     } elsif ($machine eq 'virt') {
3254         $machine = "virt-$pin_version";
3255     } else {
3256         warn "unknown machine type '$machine', not touching that!\n";
3257     }
3258
3259     return $machine;
3260 }
3261
3262 sub get_vm_machine {
3263     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3264
3265     my $machine = $forcemachine || $conf->{machine};
3266
3267     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3268         $kvmversion //= kvm_user_version();
3269         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3270         # layout which confuses windows quite a bit and may result in various regressions..
3271         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3272         if (windows_version($conf->{ostype})) {
3273             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3274         }
3275         $arch //= 'x86_64';
3276         $machine ||= $default_machines->{$arch};
3277         if ($add_pve_version) {
3278             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3279             $machine .= "+pve$pvever";
3280         }
3281     }
3282
3283     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3284         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3285         $machine = $1 if $is_pxe;
3286
3287         # for version-pinned machines that do not include a pve-version (e.g.
3288         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3289         $machine .= '+pve0';
3290
3291         $machine .= '.pxe' if $is_pxe;
3292     }
3293
3294     return $machine;
3295 }
3296
3297 sub get_ovmf_files($$$) {
3298     my ($arch, $efidisk, $smm) = @_;
3299
3300     my $types = $OVMF->{$arch}
3301         or die "no OVMF images known for architecture '$arch'\n";
3302
3303     my $type = 'default';
3304     if ($arch eq 'x86_64') {
3305         if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3306             $type = $smm ? "4m" : "4m-no-smm";
3307             $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3308         } else {
3309             # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3310         }
3311     }
3312
3313     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3314     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3315     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3316
3317     return ($ovmf_code, $ovmf_vars);
3318 }
3319
3320 my $Arch2Qemu = {
3321     aarch64 => '/usr/bin/qemu-system-aarch64',
3322     x86_64 => '/usr/bin/qemu-system-x86_64',
3323 };
3324 sub get_command_for_arch($) {
3325     my ($arch) = @_;
3326     return '/usr/bin/kvm' if is_native($arch);
3327
3328     my $cmd = $Arch2Qemu->{$arch}
3329         or die "don't know how to emulate architecture '$arch'\n";
3330     return $cmd;
3331 }
3332
3333 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3334 # to use in a QEMU command line (-cpu element), first array_intersect the result
3335 # of query_supported_ with query_understood_. This is necessary because:
3336 #
3337 # a) query_understood_ returns flags the host cannot use and
3338 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3339 #    flags, but CPU settings - with most of them being flags. Those settings
3340 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3341 #
3342 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3343 # expensive. If you need the value returned from this, you can get it much
3344 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3345 # $accel being 'kvm' or 'tcg'.
3346 #
3347 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3348 # changes, automatically populating pmxcfs.
3349 #
3350 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3351 # since kvm and tcg machines support different flags
3352 #
3353 sub query_supported_cpu_flags {
3354     my ($arch) = @_;
3355
3356     $arch //= get_host_arch();
3357     my $default_machine = $default_machines->{$arch};
3358
3359     my $flags = {};
3360
3361     # FIXME: Once this is merged, the code below should work for ARM as well:
3362     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3363     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3364         $arch eq "aarch64";
3365
3366     my $kvm_supported = defined(kvm_version());
3367     my $qemu_cmd = get_command_for_arch($arch);
3368     my $fakevmid = -1;
3369     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3370
3371     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3372     my $query_supported_run_qemu = sub {
3373         my ($kvm) = @_;
3374
3375         my $flags = {};
3376         my $cmd = [
3377             $qemu_cmd,
3378             '-machine', $default_machine,
3379             '-display', 'none',
3380             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3381             '-mon', 'chardev=qmp,mode=control',
3382             '-pidfile', $pidfile,
3383             '-S', '-daemonize'
3384         ];
3385
3386         if (!$kvm) {
3387             push @$cmd, '-accel', 'tcg';
3388         }
3389
3390         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3391         die "QEMU flag querying VM exited with code " . $rc if $rc;
3392
3393         eval {
3394             my $cmd_result = mon_cmd(
3395                 $fakevmid,
3396                 'query-cpu-model-expansion',
3397                 type => 'full',
3398                 model => { name => 'host' }
3399             );
3400
3401             my $props = $cmd_result->{model}->{props};
3402             foreach my $prop (keys %$props) {
3403                 next if $props->{$prop} ne '1';
3404                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3405                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3406                 # We only keep those with underscores, to match /proc/cpuinfo
3407                 $prop =~ s/\.|-/_/g;
3408                 $flags->{$prop} = 1;
3409             }
3410         };
3411         my $err = $@;
3412
3413         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3414         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3415
3416         die $err if $err;
3417
3418         return [ sort keys %$flags ];
3419     };
3420
3421     # We need to query QEMU twice, since KVM and TCG have different supported flags
3422     PVE::QemuConfig->lock_config($fakevmid, sub {
3423         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3424         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3425
3426         if ($kvm_supported) {
3427             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3428             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3429         }
3430     });
3431
3432     return $flags;
3433 }
3434
3435 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3436 my $understood_cpu_flag_dir = "/usr/share/kvm";
3437 sub query_understood_cpu_flags {
3438     my $arch = get_host_arch();
3439     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3440
3441     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3442         if ! -e $filepath;
3443
3444     my $raw = file_get_contents($filepath);
3445     $raw =~ s/^\s+|\s+$//g;
3446     my @flags = split(/\s+/, $raw);
3447
3448     return \@flags;
3449 }
3450
3451 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3452 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3453 my sub should_disable_smm {
3454     my ($conf, $vga, $machine) = @_;
3455
3456     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3457
3458     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3459         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3460 }
3461
3462 my sub print_ovmf_drive_commandlines {
3463     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3464
3465     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3466
3467     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3468
3469     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3470     if ($d) {
3471         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3472         my ($path, $format) = $d->@{'file', 'format'};
3473         if ($storeid) {
3474             $path = PVE::Storage::path($storecfg, $d->{file});
3475             if (!defined($format)) {
3476                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3477                 $format = qemu_img_format($scfg, $volname);
3478             }
3479         } elsif (!defined($format)) {
3480             die "efidisk format must be specified\n";
3481         }
3482         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3483         if ($path =~ m/^rbd:/) {
3484             $var_drive_str .= ',cache=writeback';
3485             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3486         }
3487         $var_drive_str .= ",format=$format,file=$path";
3488
3489         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3490         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3491     } else {
3492         log_warn("no efidisk configured! Using temporary efivars disk.");
3493         my $path = "/tmp/$vmid-ovmf.fd";
3494         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3495         $var_drive_str .= ",format=raw,file=$path";
3496         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3497     }
3498
3499     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3500 }
3501
3502 sub config_to_command {
3503     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3504         $pbs_backing) = @_;
3505
3506     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3507     my $devices = [];
3508     my $bridges = {};
3509     my $ostype = $conf->{ostype};
3510     my $winversion = windows_version($ostype);
3511     my $kvm = $conf->{kvm};
3512     my $nodename = nodename();
3513
3514     my $arch = get_vm_arch($conf);
3515     my $kvm_binary = get_command_for_arch($arch);
3516     my $kvmver = kvm_user_version($kvm_binary);
3517
3518     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3519         $kvmver //= "undefined";
3520         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3521     }
3522
3523     my $add_pve_version = min_version($kvmver, 4, 1);
3524
3525     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3526     my $machine_version = extract_version($machine_type, $kvmver);
3527     $kvm //= 1 if is_native($arch);
3528
3529     $machine_version =~ m/(\d+)\.(\d+)/;
3530     my ($machine_major, $machine_minor) = ($1, $2);
3531
3532     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3533         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3534     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3535         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3536             ." please upgrade node '$nodename'\n"
3537     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3538         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3539         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3540             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3541             ." node '$nodename'\n";
3542     }
3543
3544     # if a specific +pve version is required for a feature, use $version_guard
3545     # instead of min_version to allow machines to be run with the minimum
3546     # required version
3547     my $required_pve_version = 0;
3548     my $version_guard = sub {
3549         my ($major, $minor, $pve) = @_;
3550         return 0 if !min_version($machine_version, $major, $minor, $pve);
3551         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3552         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3553         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3554         return 1;
3555     };
3556
3557     if ($kvm && !defined kvm_version()) {
3558         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3559             ." or enable in BIOS.\n";
3560     }
3561
3562     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3563     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3564     my $use_old_bios_files = undef;
3565     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3566
3567     my $cmd = [];
3568     if ($conf->{affinity}) {
3569         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3570     }
3571
3572     push @$cmd, $kvm_binary;
3573
3574     push @$cmd, '-id', $vmid;
3575
3576     my $vmname = $conf->{name} || "vm$vmid";
3577
3578     push @$cmd, '-name', "$vmname,debug-threads=on";
3579
3580     push @$cmd, '-no-shutdown';
3581
3582     my $use_virtio = 0;
3583
3584     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3585     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3586     push @$cmd, '-mon', "chardev=qmp,mode=control";
3587
3588     if (min_version($machine_version, 2, 12)) {
3589         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3590         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3591     }
3592
3593     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3594
3595     push @$cmd, '-daemonize';
3596
3597     if ($conf->{smbios1}) {
3598         my $smbios_conf = parse_smbios1($conf->{smbios1});
3599         if ($smbios_conf->{base64}) {
3600             # Do not pass base64 flag to qemu
3601             delete $smbios_conf->{base64};
3602             my $smbios_string = "";
3603             foreach my $key (keys %$smbios_conf) {
3604                 my $value;
3605                 if ($key eq "uuid") {
3606                     $value = $smbios_conf->{uuid}
3607                 } else {
3608                     $value = decode_base64($smbios_conf->{$key});
3609                 }
3610                 # qemu accepts any binary data, only commas need escaping by double comma
3611                 $value =~ s/,/,,/g;
3612                 $smbios_string .= "," . $key . "=" . $value if $value;
3613             }
3614             push @$cmd, '-smbios', "type=1" . $smbios_string;
3615         } else {
3616             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3617         }
3618     }
3619
3620     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3621         my ($code_drive_str, $var_drive_str) =
3622             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3623         push $cmd->@*, '-drive', $code_drive_str;
3624         push $cmd->@*, '-drive', $var_drive_str;
3625     }
3626
3627     if ($q35) { # tell QEMU to load q35 config early
3628         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3629         if (min_version($machine_version, 4, 0)) {
3630             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3631         } else {
3632             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3633         }
3634     }
3635
3636     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3637         push @$cmd, $fixups->@*;
3638     }
3639
3640     if ($conf->{vmgenid}) {
3641         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3642     }
3643
3644     # add usb controllers
3645     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3646         $conf, $bridges, $arch, $machine_type, $machine_version);
3647     push @$devices, @usbcontrollers if @usbcontrollers;
3648     my $vga = parse_vga($conf->{vga});
3649
3650     my $qxlnum = vga_conf_has_spice($conf->{vga});
3651     $vga->{type} = 'qxl' if $qxlnum;
3652
3653     if (!$vga->{type}) {
3654         if ($arch eq 'aarch64') {
3655             $vga->{type} = 'virtio';
3656         } elsif (min_version($machine_version, 2, 9)) {
3657             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3658         } else {
3659             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3660         }
3661     }
3662
3663     # enable absolute mouse coordinates (needed by vnc)
3664     my $tablet = $conf->{tablet};
3665     if (!defined($tablet)) {
3666         $tablet = $defaults->{tablet};
3667         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3668         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3669     }
3670
3671     if ($tablet) {
3672         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3673         my $kbd = print_keyboarddevice_full($conf, $arch);
3674         push @$devices, '-device', $kbd if defined($kbd);
3675     }
3676
3677     my $bootorder = device_bootorder($conf);
3678
3679     # host pci device passthrough
3680     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3681         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3682
3683     # usb devices
3684     my $usb_dev_features = {};
3685     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3686
3687     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3688         $conf, $usb_dev_features, $bootorder, $machine_version);
3689     push @$devices, @usbdevices if @usbdevices;
3690
3691     # serial devices
3692     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3693         my $path = $conf->{"serial$i"} or next;
3694         if ($path eq 'socket') {
3695             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3696             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3697             # On aarch64, serial0 is the UART device. QEMU only allows
3698             # connecting UART devices via the '-serial' command line, as
3699             # the device has a fixed slot on the hardware...
3700             if ($arch eq 'aarch64' && $i == 0) {
3701                 push @$devices, '-serial', "chardev:serial$i";
3702             } else {
3703                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3704             }
3705         } else {
3706             die "no such serial device\n" if ! -c $path;
3707             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3708             push @$devices, '-device', "isa-serial,chardev=serial$i";
3709         }
3710     }
3711
3712     # parallel devices
3713     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3714         if (my $path = $conf->{"parallel$i"}) {
3715             die "no such parallel device\n" if ! -c $path;
3716             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3717             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3718             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3719         }
3720     }
3721
3722     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3723         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3724         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3725         push @$devices, @$audio_devs;
3726     }
3727
3728     # Add a TPM only if the VM is not a template,
3729     # to support backing up template VMs even if the TPM disk is write-protected.
3730     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3731
3732     my $sockets = 1;
3733     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3734     $sockets = $conf->{sockets} if  $conf->{sockets};
3735
3736     my $cores = $conf->{cores} || 1;
3737
3738     my $maxcpus = $sockets * $cores;
3739
3740     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3741
3742     my $allowed_vcpus = $cpuinfo->{cpus};
3743
3744     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3745
3746     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3747         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3748         for (my $i = 2; $i <= $vcpus; $i++)  {
3749             my $cpustr = print_cpu_device($conf,$i);
3750             push @$cmd, '-device', $cpustr;
3751         }
3752
3753     } else {
3754
3755         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3756     }
3757     push @$cmd, '-nodefaults';
3758
3759     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3760
3761     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3762
3763     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3764
3765     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3766         push @$devices, '-device', print_vga_device(
3767             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3768
3769         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3770
3771         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3772         push @$cmd,  '-vnc', "unix:$socket,password=on";
3773     } else {
3774         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3775         push @$cmd, '-nographic';
3776     }
3777
3778     # time drift fix
3779     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3780     my $useLocaltime = $conf->{localtime};
3781
3782     if ($winversion >= 5) { # windows
3783         $useLocaltime = 1 if !defined($conf->{localtime});
3784
3785         # use time drift fix when acpi is enabled
3786         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3787             $tdf = 1 if !defined($conf->{tdf});
3788         }
3789     }
3790
3791     if ($winversion >= 6) {
3792         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3793         push @$machineFlags, 'hpet=off';
3794     }
3795
3796     push @$rtcFlags, 'driftfix=slew' if $tdf;
3797
3798     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3799         push @$rtcFlags, "base=$conf->{startdate}";
3800     } elsif ($useLocaltime) {
3801         push @$rtcFlags, 'base=localtime';
3802     }
3803
3804     if ($forcecpu) {
3805         push @$cmd, '-cpu', $forcecpu;
3806     } else {
3807         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3808     }
3809
3810     PVE::QemuServer::Memory::config(
3811         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3812
3813     push @$cmd, '-S' if $conf->{freeze};
3814
3815     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3816
3817     my $guest_agent = parse_guest_agent($conf);
3818
3819     if ($guest_agent->{enabled}) {
3820         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3821         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3822
3823         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3824             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3825             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3826             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3827         } elsif ($guest_agent->{type} eq 'isa') {
3828             push @$devices, '-device', "isa-serial,chardev=qga0";
3829         }
3830     }
3831
3832     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3833     if ($rng && $version_guard->(4, 1, 2)) {
3834         check_rng_source($rng->{source});
3835
3836         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3837         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3838         my $limiter_str = "";
3839         if ($max_bytes) {
3840             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3841         }
3842
3843         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3844         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3845         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3846     }
3847
3848     my $spice_port;
3849
3850     assert_clipboard_config($vga);
3851     my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3852
3853     if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3854         if ($qxlnum > 1) {
3855             if ($winversion){
3856                 for (my $i = 1; $i < $qxlnum; $i++){
3857                     push @$devices, '-device', print_vga_device(
3858                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3859                 }
3860             } else {
3861                 # assume other OS works like Linux
3862                 my ($ram, $vram) = ("134217728", "67108864");
3863                 if ($vga->{memory}) {
3864                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3865                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3866                 }
3867                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3868                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3869             }
3870         }
3871
3872         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3873
3874         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3875         if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3876             push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3877         } else {
3878             push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3879         }
3880         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3881
3882         if ($is_spice) {
3883             my $pfamily = PVE::Tools::get_host_address_family($nodename);
3884             my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3885             die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3886
3887             my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3888             $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3889
3890             my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3891             my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3892             if ($spice_enhancement->{foldersharing}) {
3893                 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3894                 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3895             }
3896
3897             my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3898             $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3899                 if $spice_enhancement->{videostreaming};
3900             push @$devices, '-spice', "$spice_opts";
3901         }
3902     }
3903
3904     # enable balloon by default, unless explicitly disabled
3905     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3906         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3907         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3908         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3909         push @$devices, '-device', $ballooncmd;
3910     }
3911
3912     if ($conf->{watchdog}) {
3913         my $wdopts = parse_watchdog($conf->{watchdog});
3914         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3915         my $watchdog = $wdopts->{model} || 'i6300esb';
3916         push @$devices, '-device', "$watchdog$pciaddr";
3917         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3918     }
3919
3920     my $vollist = [];
3921     my $scsicontroller = {};
3922     my $ahcicontroller = {};
3923     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3924
3925     # Add iscsi initiator name if available
3926     if (my $initiator = get_initiator_name()) {
3927         push @$devices, '-iscsi', "initiator-name=$initiator";
3928     }
3929
3930     PVE::QemuConfig->foreach_volume($conf, sub {
3931         my ($ds, $drive) = @_;
3932
3933         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3934             check_volume_storage_type($storecfg, $drive->{file});
3935             push @$vollist, $drive->{file};
3936         }
3937
3938         # ignore efidisk here, already added in bios/fw handling code above
3939         return if $drive->{interface} eq 'efidisk';
3940         # similar for TPM
3941         return if $drive->{interface} eq 'tpmstate';
3942
3943         $use_virtio = 1 if $ds =~ m/^virtio/;
3944
3945         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3946
3947         if ($drive->{interface} eq 'virtio'){
3948            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3949         }
3950
3951         if ($drive->{interface} eq 'scsi') {
3952
3953             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3954
3955             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3956                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3957
3958             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3959             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3960
3961             my $iothread = '';
3962             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3963                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3964                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3965             } elsif ($drive->{iothread}) {
3966                 log_warn(
3967                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3968                 );
3969             }
3970
3971             my $queues = '';
3972             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3973                 $queues = ",num_queues=$drive->{queues}";
3974             }
3975
3976             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3977                 if !$scsicontroller->{$controller};
3978             $scsicontroller->{$controller}=1;
3979         }
3980
3981         if ($drive->{interface} eq 'sata') {
3982             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3983             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3984             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3985                 if !$ahcicontroller->{$controller};
3986             $ahcicontroller->{$controller}=1;
3987         }
3988
3989         my $pbs_conf = $pbs_backing->{$ds};
3990         my $pbs_name = undef;
3991         if ($pbs_conf) {
3992             $pbs_name = "drive-$ds-pbs";
3993             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3994         }
3995
3996         my $drive_cmd = print_drive_commandline_full(
3997             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3998
3999         # extra protection for templates, but SATA and IDE don't support it..
4000         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4001
4002         push @$devices, '-drive',$drive_cmd;
4003         push @$devices, '-device', print_drivedevice_full(
4004             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4005     });
4006
4007     for (my $i = 0; $i < $MAX_NETS; $i++) {
4008         my $netname = "net$i";
4009
4010         next if !$conf->{$netname};
4011         my $d = parse_net($conf->{$netname});
4012         next if !$d;
4013         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4014
4015         $use_virtio = 1 if $d->{model} eq 'virtio';
4016
4017         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4018
4019         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4020         push @$devices, '-netdev', $netdevfull;
4021
4022         my $netdevicefull = print_netdevice_full(
4023             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4024
4025         push @$devices, '-device', $netdevicefull;
4026     }
4027
4028     if ($conf->{ivshmem}) {
4029         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4030
4031         my $bus;
4032         if ($q35) {
4033             $bus = print_pcie_addr("ivshmem");
4034         } else {
4035             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4036         }
4037
4038         my $ivshmem_name = $ivshmem->{name} // $vmid;
4039         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4040
4041         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4042         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4043             .",size=$ivshmem->{size}M";
4044     }
4045
4046     # pci.4 is nested in pci.1
4047     $bridges->{1} = 1 if $bridges->{4};
4048
4049     if (!$q35) { # add pci bridges
4050         if (min_version($machine_version, 2, 3)) {
4051            $bridges->{1} = 1;
4052            $bridges->{2} = 1;
4053         }
4054         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4055     }
4056
4057     for my $k (sort {$b cmp $a} keys %$bridges) {
4058         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4059
4060         my $k_name = $k;
4061         if ($k == 2 && $legacy_igd) {
4062             $k_name = "$k-igd";
4063         }
4064         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4065         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4066
4067         if ($q35) { # add after -readconfig pve-q35.cfg
4068             splice @$devices, 2, 0, '-device', $devstr;
4069         } else {
4070             unshift @$devices, '-device', $devstr if $k > 0;
4071         }
4072     }
4073
4074     if (!$kvm) {
4075         push @$machineFlags, 'accel=tcg';
4076     }
4077
4078     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4079
4080     my $machine_type_min = $machine_type;
4081     if ($add_pve_version) {
4082         $machine_type_min =~ s/\+pve\d+$//;
4083         $machine_type_min .= "+pve$required_pve_version";
4084     }
4085     push @$machineFlags, "type=${machine_type_min}";
4086
4087     push @$cmd, @$devices;
4088     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4089     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4090     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4091
4092     if (my $vmstate = $conf->{vmstate}) {
4093         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4094         push @$vollist, $vmstate;
4095         push @$cmd, '-loadstate', $statepath;
4096         print "activating and using '$vmstate' as vmstate\n";
4097     }
4098
4099     if (PVE::QemuConfig->is_template($conf)) {
4100         # needed to workaround base volumes being read-only
4101         push @$cmd, '-snapshot';
4102     }
4103
4104     # add custom args
4105     if ($conf->{args}) {
4106         my $aa = PVE::Tools::split_args($conf->{args});
4107         push @$cmd, @$aa;
4108     }
4109
4110     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4111 }
4112
4113 sub check_rng_source {
4114     my ($source) = @_;
4115
4116     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4117     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4118         if ! -e $source;
4119
4120     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4121     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4122         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4123         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4124         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4125             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4126             ." to the host.\n";
4127     }
4128 }
4129
4130 sub spice_port {
4131     my ($vmid) = @_;
4132
4133     my $res = mon_cmd($vmid, 'query-spice');
4134
4135     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4136 }
4137
4138 sub vm_devices_list {
4139     my ($vmid) = @_;
4140
4141     my $res = mon_cmd($vmid, 'query-pci');
4142     my $devices_to_check = [];
4143     my $devices = {};
4144     foreach my $pcibus (@$res) {
4145         push @$devices_to_check, @{$pcibus->{devices}},
4146     }
4147
4148     while (@$devices_to_check) {
4149         my $to_check = [];
4150         for my $d (@$devices_to_check) {
4151             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4152             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4153
4154             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4155             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4156         }
4157         $devices_to_check = $to_check;
4158     }
4159
4160     my $resblock = mon_cmd($vmid, 'query-block');
4161     foreach my $block (@$resblock) {
4162         if($block->{device} =~ m/^drive-(\S+)/){
4163                 $devices->{$1} = 1;
4164         }
4165     }
4166
4167     my $resmice = mon_cmd($vmid, 'query-mice');
4168     foreach my $mice (@$resmice) {
4169         if ($mice->{name} eq 'QEMU HID Tablet') {
4170             $devices->{tablet} = 1;
4171             last;
4172         }
4173     }
4174
4175     # for usb devices there is no query-usb
4176     # but we can iterate over the entries in
4177     # qom-list path=/machine/peripheral
4178     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4179     foreach my $per (@$resperipheral) {
4180         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4181             $devices->{$per->{name}} = 1;
4182         }
4183     }
4184
4185     return $devices;
4186 }
4187
4188 sub vm_deviceplug {
4189     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4190
4191     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4192
4193     my $devices_list = vm_devices_list($vmid);
4194     return 1 if defined($devices_list->{$deviceid});
4195
4196     # add PCI bridge if we need it for the device
4197     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4198
4199     if ($deviceid eq 'tablet') {
4200         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4201     } elsif ($deviceid eq 'keyboard') {
4202         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4203     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4204         my $id = $1;
4205         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4206         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4207     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4208         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4209     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4210         qemu_iothread_add($vmid, $deviceid, $device);
4211
4212         qemu_driveadd($storecfg, $vmid, $device);
4213         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4214
4215         qemu_deviceadd($vmid, $devicefull);
4216         eval { qemu_deviceaddverify($vmid, $deviceid); };
4217         if (my $err = $@) {
4218             eval { qemu_drivedel($vmid, $deviceid); };
4219             warn $@ if $@;
4220             die $err;
4221         }
4222     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4223         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4224         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4225         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4226
4227         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4228
4229         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4230             qemu_iothread_add($vmid, $deviceid, $device);
4231             $devicefull .= ",iothread=iothread-$deviceid";
4232         }
4233
4234         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4235             $devicefull .= ",num_queues=$device->{queues}";
4236         }
4237
4238         qemu_deviceadd($vmid, $devicefull);
4239         qemu_deviceaddverify($vmid, $deviceid);
4240     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4241         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4242         qemu_driveadd($storecfg, $vmid, $device);
4243
4244         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4245         eval { qemu_deviceadd($vmid, $devicefull); };
4246         if (my $err = $@) {
4247             eval { qemu_drivedel($vmid, $deviceid); };
4248             warn $@ if $@;
4249             die $err;
4250         }
4251     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4252         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4253
4254         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4255         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4256         my $use_old_bios_files = undef;
4257         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4258
4259         my $netdevicefull = print_netdevice_full(
4260             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4261         qemu_deviceadd($vmid, $netdevicefull);
4262         eval {
4263             qemu_deviceaddverify($vmid, $deviceid);
4264             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4265         };
4266         if (my $err = $@) {
4267             eval { qemu_netdevdel($vmid, $deviceid); };
4268             warn $@ if $@;
4269             die $err;
4270         }
4271     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4272         my $bridgeid = $2;
4273         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4274         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4275
4276         qemu_deviceadd($vmid, $devicefull);
4277         qemu_deviceaddverify($vmid, $deviceid);
4278     } else {
4279         die "can't hotplug device '$deviceid'\n";
4280     }
4281
4282     return 1;
4283 }
4284
4285 # fixme: this should raise exceptions on error!
4286 sub vm_deviceunplug {
4287     my ($vmid, $conf, $deviceid) = @_;
4288
4289     my $devices_list = vm_devices_list($vmid);
4290     return 1 if !defined($devices_list->{$deviceid});
4291
4292     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4293     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4294
4295     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4296         qemu_devicedel($vmid, $deviceid);
4297     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4298         qemu_devicedel($vmid, $deviceid);
4299         qemu_devicedelverify($vmid, $deviceid);
4300     } elsif ($deviceid =~ m/^usb\d+$/) {
4301         qemu_devicedel($vmid, $deviceid);
4302         qemu_devicedelverify($vmid, $deviceid);
4303     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4304         my $device = parse_drive($deviceid, $conf->{$deviceid});
4305
4306         qemu_devicedel($vmid, $deviceid);
4307         qemu_devicedelverify($vmid, $deviceid);
4308         qemu_drivedel($vmid, $deviceid);
4309         qemu_iothread_del($vmid, $deviceid, $device);
4310     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4311         qemu_devicedel($vmid, $deviceid);
4312         qemu_devicedelverify($vmid, $deviceid);
4313     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4314         my $device = parse_drive($deviceid, $conf->{$deviceid});
4315
4316         qemu_devicedel($vmid, $deviceid);
4317         qemu_devicedelverify($vmid, $deviceid);
4318         qemu_drivedel($vmid, $deviceid);
4319         qemu_deletescsihw($conf, $vmid, $deviceid);
4320
4321         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4322             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4323     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4324         qemu_devicedel($vmid, $deviceid);
4325         qemu_devicedelverify($vmid, $deviceid);
4326         qemu_netdevdel($vmid, $deviceid);
4327     } else {
4328         die "can't unplug device '$deviceid'\n";
4329     }
4330
4331     return 1;
4332 }
4333
4334 sub qemu_spice_usbredir_chardev_add {
4335     my ($vmid, $id) = @_;
4336
4337     mon_cmd($vmid, "chardev-add" , (
4338         id => $id,
4339         backend => {
4340             type => 'spicevmc',
4341             data => {
4342                 type => "usbredir",
4343             },
4344         },
4345     ));
4346 }
4347
4348 sub qemu_iothread_add {
4349     my ($vmid, $deviceid, $device) = @_;
4350
4351     if ($device->{iothread}) {
4352         my $iothreads = vm_iothreads_list($vmid);
4353         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4354     }
4355 }
4356
4357 sub qemu_iothread_del {
4358     my ($vmid, $deviceid, $device) = @_;
4359
4360     if ($device->{iothread}) {
4361         my $iothreads = vm_iothreads_list($vmid);
4362         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4363     }
4364 }
4365
4366 sub qemu_driveadd {
4367     my ($storecfg, $vmid, $device) = @_;
4368
4369     my $kvmver = get_running_qemu_version($vmid);
4370     my $io_uring = min_version($kvmver, 6, 0);
4371     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4372     $drive =~ s/\\/\\\\/g;
4373     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4374
4375     # If the command succeeds qemu prints: "OK"
4376     return 1 if $ret =~ m/OK/s;
4377
4378     die "adding drive failed: $ret\n";
4379 }
4380
4381 sub qemu_drivedel {
4382     my ($vmid, $deviceid) = @_;
4383
4384     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4385     $ret =~ s/^\s+//;
4386
4387     return 1 if $ret eq "";
4388
4389     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4390     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4391
4392     die "deleting drive $deviceid failed : $ret\n";
4393 }
4394
4395 sub qemu_deviceaddverify {
4396     my ($vmid, $deviceid) = @_;
4397
4398     for (my $i = 0; $i <= 5; $i++) {
4399          my $devices_list = vm_devices_list($vmid);
4400          return 1 if defined($devices_list->{$deviceid});
4401          sleep 1;
4402     }
4403
4404     die "error on hotplug device '$deviceid'\n";
4405 }
4406
4407
4408 sub qemu_devicedelverify {
4409     my ($vmid, $deviceid) = @_;
4410
4411     # need to verify that the device is correctly removed as device_del
4412     # is async and empty return is not reliable
4413
4414     for (my $i = 0; $i <= 5; $i++) {
4415          my $devices_list = vm_devices_list($vmid);
4416          return 1 if !defined($devices_list->{$deviceid});
4417          sleep 1;
4418     }
4419
4420     die "error on hot-unplugging device '$deviceid'\n";
4421 }
4422
4423 sub qemu_findorcreatescsihw {
4424     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4425
4426     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4427
4428     my $scsihwid="$controller_prefix$controller";
4429     my $devices_list = vm_devices_list($vmid);
4430
4431     if (!defined($devices_list->{$scsihwid})) {
4432         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4433     }
4434
4435     return 1;
4436 }
4437
4438 sub qemu_deletescsihw {
4439     my ($conf, $vmid, $opt) = @_;
4440
4441     my $device = parse_drive($opt, $conf->{$opt});
4442
4443     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4444         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4445         return 1;
4446     }
4447
4448     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4449
4450     my $devices_list = vm_devices_list($vmid);
4451     foreach my $opt (keys %{$devices_list}) {
4452         if (is_valid_drivename($opt)) {
4453             my $drive = parse_drive($opt, $conf->{$opt});
4454             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4455                 return 1;
4456             }
4457         }
4458     }
4459
4460     my $scsihwid="scsihw$controller";
4461
4462     vm_deviceunplug($vmid, $conf, $scsihwid);
4463
4464     return 1;
4465 }
4466
4467 sub qemu_add_pci_bridge {
4468     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4469
4470     my $bridges = {};
4471
4472     my $bridgeid;
4473
4474     print_pci_addr($device, $bridges, $arch, $machine_type);
4475
4476     while (my ($k, $v) = each %$bridges) {
4477         $bridgeid = $k;
4478     }
4479     return 1 if !defined($bridgeid) || $bridgeid < 1;
4480
4481     my $bridge = "pci.$bridgeid";
4482     my $devices_list = vm_devices_list($vmid);
4483
4484     if (!defined($devices_list->{$bridge})) {
4485         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4486     }
4487
4488     return 1;
4489 }
4490
4491 sub qemu_set_link_status {
4492     my ($vmid, $device, $up) = @_;
4493
4494     mon_cmd($vmid, "set_link", name => $device,
4495                up => $up ? JSON::true : JSON::false);
4496 }
4497
4498 sub qemu_netdevadd {
4499     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4500
4501     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4502     my %options =  split(/[=,]/, $netdev);
4503
4504     if (defined(my $vhost = $options{vhost})) {
4505         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4506     }
4507
4508     if (defined(my $queues = $options{queues})) {
4509         $options{queues} = $queues + 0;
4510     }
4511
4512     mon_cmd($vmid, "netdev_add",  %options);
4513     return 1;
4514 }
4515
4516 sub qemu_netdevdel {
4517     my ($vmid, $deviceid) = @_;
4518
4519     mon_cmd($vmid, "netdev_del", id => $deviceid);
4520 }
4521
4522 sub qemu_usb_hotplug {
4523     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4524
4525     return if !$device;
4526
4527     # remove the old one first
4528     vm_deviceunplug($vmid, $conf, $deviceid);
4529
4530     # check if xhci controller is necessary and available
4531     my $devicelist = vm_devices_list($vmid);
4532
4533     if (!$devicelist->{xhci}) {
4534         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4535         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4536     }
4537
4538     # add the new one
4539     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4540 }
4541
4542 sub qemu_cpu_hotplug {
4543     my ($vmid, $conf, $vcpus) = @_;
4544
4545     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4546
4547     my $sockets = 1;
4548     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4549     $sockets = $conf->{sockets} if  $conf->{sockets};
4550     my $cores = $conf->{cores} || 1;
4551     my $maxcpus = $sockets * $cores;
4552
4553     $vcpus = $maxcpus if !$vcpus;
4554
4555     die "you can't add more vcpus than maxcpus\n"
4556         if $vcpus > $maxcpus;
4557
4558     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4559
4560     if ($vcpus < $currentvcpus) {
4561
4562         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4563
4564             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4565                 qemu_devicedel($vmid, "cpu$i");
4566                 my $retry = 0;
4567                 my $currentrunningvcpus = undef;
4568                 while (1) {
4569                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4570                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4571                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4572                     $retry++;
4573                     sleep 1;
4574                 }
4575                 #update conf after each succesfull cpu unplug
4576                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4577                 PVE::QemuConfig->write_config($vmid, $conf);
4578             }
4579         } else {
4580             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4581         }
4582
4583         return;
4584     }
4585
4586     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4587     die "vcpus in running vm does not match its configuration\n"
4588         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4589
4590     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4591
4592         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4593             my $cpustr = print_cpu_device($conf, $i);
4594             qemu_deviceadd($vmid, $cpustr);
4595
4596             my $retry = 0;
4597             my $currentrunningvcpus = undef;
4598             while (1) {
4599                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4600                 last if scalar(@{$currentrunningvcpus}) == $i;
4601                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4602                 sleep 1;
4603                 $retry++;
4604             }
4605             #update conf after each succesfull cpu hotplug
4606             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4607             PVE::QemuConfig->write_config($vmid, $conf);
4608         }
4609     } else {
4610
4611         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4612             mon_cmd($vmid, "cpu-add", id => int($i));
4613         }
4614     }
4615 }
4616
4617 sub qemu_block_set_io_throttle {
4618     my ($vmid, $deviceid,
4619         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4620         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4621         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4622         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4623
4624     return if !check_running($vmid) ;
4625
4626     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4627         bps => int($bps),
4628         bps_rd => int($bps_rd),
4629         bps_wr => int($bps_wr),
4630         iops => int($iops),
4631         iops_rd => int($iops_rd),
4632         iops_wr => int($iops_wr),
4633         bps_max => int($bps_max),
4634         bps_rd_max => int($bps_rd_max),
4635         bps_wr_max => int($bps_wr_max),
4636         iops_max => int($iops_max),
4637         iops_rd_max => int($iops_rd_max),
4638         iops_wr_max => int($iops_wr_max),
4639         bps_max_length => int($bps_max_length),
4640         bps_rd_max_length => int($bps_rd_max_length),
4641         bps_wr_max_length => int($bps_wr_max_length),
4642         iops_max_length => int($iops_max_length),
4643         iops_rd_max_length => int($iops_rd_max_length),
4644         iops_wr_max_length => int($iops_wr_max_length),
4645     );
4646
4647 }
4648
4649 sub qemu_block_resize {
4650     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4651
4652     my $running = check_running($vmid);
4653
4654     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4655
4656     return if !$running;
4657
4658     my $padding = (1024 - $size % 1024) % 1024;
4659     $size = $size + $padding;
4660
4661     mon_cmd(
4662         $vmid,
4663         "block_resize",
4664         device => $deviceid,
4665         size => int($size),
4666         timeout => 60,
4667     );
4668 }
4669
4670 sub qemu_volume_snapshot {
4671     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4672
4673     my $running = check_running($vmid);
4674
4675     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4676         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4677     } else {
4678         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4679     }
4680 }
4681
4682 sub qemu_volume_snapshot_delete {
4683     my ($vmid, $storecfg, $volid, $snap) = @_;
4684
4685     my $running = check_running($vmid);
4686     my $attached_deviceid;
4687
4688     if ($running) {
4689         my $conf = PVE::QemuConfig->load_config($vmid);
4690         PVE::QemuConfig->foreach_volume($conf, sub {
4691             my ($ds, $drive) = @_;
4692             $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4693         });
4694     }
4695
4696     if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4697         mon_cmd(
4698             $vmid,
4699             'blockdev-snapshot-delete-internal-sync',
4700             device => $attached_deviceid,
4701             name => $snap,
4702         );
4703     } else {
4704         PVE::Storage::volume_snapshot_delete(
4705             $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4706     }
4707 }
4708
4709 sub set_migration_caps {
4710     my ($vmid, $savevm) = @_;
4711
4712     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4713
4714     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4715     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4716
4717     my $cap_ref = [];
4718
4719     my $enabled_cap = {
4720         "auto-converge" => 1,
4721         "xbzrle" => 1,
4722         "x-rdma-pin-all" => 0,
4723         "zero-blocks" => 0,
4724         "compress" => 0,
4725         "dirty-bitmaps" => $dirty_bitmaps,
4726     };
4727
4728     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4729
4730     for my $supported_capability (@$supported_capabilities) {
4731         push @$cap_ref, {
4732             capability => $supported_capability->{capability},
4733             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4734         };
4735     }
4736
4737     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4738 }
4739
4740 sub foreach_volid {
4741     my ($conf, $func, @param) = @_;
4742
4743     my $volhash = {};
4744
4745     my $test_volid = sub {
4746         my ($key, $drive, $snapname, $pending) = @_;
4747
4748         my $volid = $drive->{file};
4749         return if !$volid;
4750
4751         $volhash->{$volid}->{cdrom} //= 1;
4752         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4753
4754         my $replicate = $drive->{replicate} // 1;
4755         $volhash->{$volid}->{replicate} //= 0;
4756         $volhash->{$volid}->{replicate} = 1 if $replicate;
4757
4758         $volhash->{$volid}->{shared} //= 0;
4759         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4760
4761         $volhash->{$volid}->{is_unused} //= 0;
4762         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4763
4764         $volhash->{$volid}->{is_attached} //= 0;
4765         $volhash->{$volid}->{is_attached} = 1
4766             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4767
4768         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4769             if defined($snapname);
4770
4771         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4772
4773         my $size = $drive->{size};
4774         $volhash->{$volid}->{size} //= $size if $size;
4775
4776         $volhash->{$volid}->{is_vmstate} //= 0;
4777         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4778
4779         $volhash->{$volid}->{is_tpmstate} //= 0;
4780         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4781
4782         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4783     };
4784
4785     my $include_opts = {
4786         extra_keys => ['vmstate'],
4787         include_unused => 1,
4788     };
4789
4790     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4791
4792     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4793         if defined($conf->{pending}) && $conf->{pending}->%*;
4794
4795     foreach my $snapname (keys %{$conf->{snapshots}}) {
4796         my $snap = $conf->{snapshots}->{$snapname};
4797         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4798     }
4799
4800     foreach my $volid (keys %$volhash) {
4801         &$func($volid, $volhash->{$volid}, @param);
4802     }
4803 }
4804
4805 my $fast_plug_option = {
4806     'description' => 1,
4807     'hookscript' => 1,
4808     'lock' => 1,
4809     'migrate_downtime' => 1,
4810     'migrate_speed' => 1,
4811     'name' => 1,
4812     'onboot' => 1,
4813     'protection' => 1,
4814     'shares' => 1,
4815     'startup' => 1,
4816     'tags' => 1,
4817     'vmstatestorage' => 1,
4818 };
4819
4820 for my $opt (keys %$confdesc_cloudinit) {
4821     $fast_plug_option->{$opt} = 1;
4822 };
4823
4824 # hotplug changes in [PENDING]
4825 # $selection hash can be used to only apply specified options, for
4826 # example: { cores => 1 } (only apply changed 'cores')
4827 # $errors ref is used to return error messages
4828 sub vmconfig_hotplug_pending {
4829     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4830
4831     my $defaults = load_defaults();
4832     my $arch = get_vm_arch($conf);
4833     my $machine_type = get_vm_machine($conf, undef, $arch);
4834
4835     # commit values which do not have any impact on running VM first
4836     # Note: those option cannot raise errors, we we do not care about
4837     # $selection and always apply them.
4838
4839     my $add_error = sub {
4840         my ($opt, $msg) = @_;
4841         $errors->{$opt} = "hotplug problem - $msg";
4842     };
4843
4844     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4845
4846     my $cloudinit_record_changed = sub {
4847         my ($conf, $opt, $old, $new) = @_;
4848         return if !$cloudinit_pending_properties->{$opt};
4849
4850         my $ci = ($conf->{cloudinit} //= {});
4851
4852         my $recorded = $ci->{$opt};
4853         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4854
4855         if (defined($new)) {
4856             if (defined($old)) {
4857                 # an existing value is being modified
4858                 if (defined($recorded)) {
4859                     # the value was already not in sync
4860                     if ($new eq $recorded) {
4861                         # a value is being reverted to the cloud-init state:
4862                         delete $ci->{$opt};
4863                         delete $added{$opt};
4864                     } else {
4865                         # the value was changed multiple times, do nothing
4866                     }
4867                 } elsif ($added{$opt}) {
4868                     # the value had been marked as added and is being changed, do nothing
4869                 } else {
4870                     # the value is new, record it:
4871                     $ci->{$opt} = $old;
4872                 }
4873             } else {
4874                 # a new value is being added
4875                 if (defined($recorded)) {
4876                     # it was already not in sync
4877                     if ($new eq $recorded) {
4878                         # a value is being reverted to the cloud-init state:
4879                         delete $ci->{$opt};
4880                         delete $added{$opt};
4881                     } else {
4882                         # the value had temporarily been removed, do nothing
4883                     }
4884                 } elsif ($added{$opt}) {
4885                     # the value had been marked as added already, do nothing
4886                 } else {
4887                     # the value is new, add it
4888                     $added{$opt} = 1;
4889                 }
4890             }
4891         } elsif (!defined($old)) {
4892             # a non-existent value is being removed? ignore...
4893         } else {
4894             # a value is being deleted
4895             if (defined($recorded)) {
4896                 # a value was already recorded, just keep it
4897             } elsif ($added{$opt}) {
4898                 # the value was marked as added, remove it
4899                 delete $added{$opt};
4900             } else {
4901                 # a previously unrecorded value is being removed, record the old value:
4902                 $ci->{$opt} = $old;
4903             }
4904         }
4905
4906         my $added = join(',', sort keys %added);
4907         $ci->{added} = $added if length($added);
4908     };
4909
4910     my $changes = 0;
4911     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4912         if ($fast_plug_option->{$opt}) {
4913             my $new = delete $conf->{pending}->{$opt};
4914             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4915             $conf->{$opt} = $new;
4916             $changes = 1;
4917         }
4918     }
4919
4920     if ($changes) {
4921         PVE::QemuConfig->write_config($vmid, $conf);
4922     }
4923
4924     my $ostype = $conf->{ostype};
4925     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4926     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4927     my $usb_hotplug = $hotplug_features->{usb}
4928         && min_version($version, 7, 1)
4929         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4930
4931     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4932     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4933
4934     foreach my $opt (sort keys %$pending_delete_hash) {
4935         next if $selection && !$selection->{$opt};
4936         my $force = $pending_delete_hash->{$opt}->{force};
4937         eval {
4938             if ($opt eq 'hotplug') {
4939                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4940             } elsif ($opt eq 'tablet') {
4941                 die "skip\n" if !$hotplug_features->{usb};
4942                 if ($defaults->{tablet}) {
4943                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4944                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4945                         if $arch eq 'aarch64';
4946                 } else {
4947                     vm_deviceunplug($vmid, $conf, 'tablet');
4948                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4949                 }
4950             } elsif ($opt =~ m/^usb(\d+)$/) {
4951                 my $index = $1;
4952                 die "skip\n" if !$usb_hotplug;
4953                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4954                 vm_deviceunplug($vmid, $conf, $opt);
4955             } elsif ($opt eq 'vcpus') {
4956                 die "skip\n" if !$hotplug_features->{cpu};
4957                 qemu_cpu_hotplug($vmid, $conf, undef);
4958             } elsif ($opt eq 'balloon') {
4959                 # enable balloon device is not hotpluggable
4960                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4961                 # here we reset the ballooning value to memory
4962                 my $balloon = get_current_memory($conf->{memory});
4963                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4964             } elsif ($fast_plug_option->{$opt}) {
4965                 # do nothing
4966             } elsif ($opt =~ m/^net(\d+)$/) {
4967                 die "skip\n" if !$hotplug_features->{network};
4968                 vm_deviceunplug($vmid, $conf, $opt);
4969                 if($have_sdn) {
4970                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
4971                     PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
4972                 }
4973             } elsif (is_valid_drivename($opt)) {
4974                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4975                 vm_deviceunplug($vmid, $conf, $opt);
4976                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4977             } elsif ($opt =~ m/^memory$/) {
4978                 die "skip\n" if !$hotplug_features->{memory};
4979                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
4980             } elsif ($opt eq 'cpuunits') {
4981                 $cgroup->change_cpu_shares(undef);
4982             } elsif ($opt eq 'cpulimit') {
4983                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4984             } else {
4985                 die "skip\n";
4986             }
4987         };
4988         if (my $err = $@) {
4989             &$add_error($opt, $err) if $err ne "skip\n";
4990         } else {
4991             my $old = delete $conf->{$opt};
4992             $cloudinit_record_changed->($conf, $opt, $old, undef);
4993             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4994         }
4995     }
4996
4997     my $cloudinit_opt;
4998     foreach my $opt (keys %{$conf->{pending}}) {
4999         next if $selection && !$selection->{$opt};
5000         my $value = $conf->{pending}->{$opt};
5001         eval {
5002             if ($opt eq 'hotplug') {
5003                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5004                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
5005             } elsif ($opt eq 'tablet') {
5006                 die "skip\n" if !$hotplug_features->{usb};
5007                 if ($value == 1) {
5008                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5009                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5010                         if $arch eq 'aarch64';
5011                 } elsif ($value == 0) {
5012                     vm_deviceunplug($vmid, $conf, 'tablet');
5013                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5014                 }
5015             } elsif ($opt =~ m/^usb(\d+)$/) {
5016                 my $index = $1;
5017                 die "skip\n" if !$usb_hotplug;
5018                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5019                 my $id = $opt;
5020                 if ($d->{host} =~ m/^spice$/i)  {
5021                     $id = "usbredirdev$index";
5022                 }
5023                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5024             } elsif ($opt eq 'vcpus') {
5025                 die "skip\n" if !$hotplug_features->{cpu};
5026                 qemu_cpu_hotplug($vmid, $conf, $value);
5027             } elsif ($opt eq 'balloon') {
5028                 # enable/disable balloning device is not hotpluggable
5029                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5030                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5031                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5032
5033                 # allow manual ballooning if shares is set to zero
5034                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5035                     my $memory = get_current_memory($conf->{memory});
5036                     my $balloon = $conf->{pending}->{balloon} || $memory;
5037                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5038                 }
5039             } elsif ($opt =~ m/^net(\d+)$/) {
5040                 # some changes can be done without hotplug
5041                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5042                                     $vmid, $opt, $value, $arch, $machine_type);
5043             } elsif (is_valid_drivename($opt)) {
5044                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5045                 # some changes can be done without hotplug
5046                 my $drive = parse_drive($opt, $value);
5047                 if (drive_is_cloudinit($drive)) {
5048                     $cloudinit_opt = [$opt, $drive];
5049                     # apply all the other changes first, then generate the cloudinit disk
5050                     die "skip\n";
5051                 }
5052                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5053                                      $vmid, $opt, $value, $arch, $machine_type);
5054             } elsif ($opt =~ m/^memory$/) { #dimms
5055                 die "skip\n" if !$hotplug_features->{memory};
5056                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5057             } elsif ($opt eq 'cpuunits') {
5058                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5059                 $cgroup->change_cpu_shares($new_cpuunits);
5060             } elsif ($opt eq 'cpulimit') {
5061                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5062                 $cgroup->change_cpu_quota($cpulimit, 100000);
5063             } elsif ($opt eq 'agent') {
5064                 vmconfig_update_agent($conf, $opt, $value);
5065             } else {
5066                 die "skip\n";  # skip non-hot-pluggable options
5067             }
5068         };
5069         if (my $err = $@) {
5070             &$add_error($opt, $err) if $err ne "skip\n";
5071         } else {
5072             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5073             $conf->{$opt} = $value;
5074             delete $conf->{pending}->{$opt};
5075         }
5076     }
5077
5078     if (defined($cloudinit_opt)) {
5079         my ($opt, $drive) = @$cloudinit_opt;
5080         my $value = $conf->{pending}->{$opt};
5081         eval {
5082             my $temp = {%$conf, $opt => $value};
5083             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5084             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5085                                  $vmid, $opt, $value, $arch, $machine_type);
5086         };
5087         if (my $err = $@) {
5088             &$add_error($opt, $err) if $err ne "skip\n";
5089         } else {
5090             $conf->{$opt} = $value;
5091             delete $conf->{pending}->{$opt};
5092         }
5093     }
5094
5095     # unplug xhci controller if no usb device is left
5096     if ($usb_hotplug) {
5097         my $has_usb = 0;
5098         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5099             next if !defined($conf->{"usb$i"});
5100             $has_usb = 1;
5101             last;
5102         }
5103         if (!$has_usb) {
5104             vm_deviceunplug($vmid, $conf, 'xhci');
5105         }
5106     }
5107
5108     PVE::QemuConfig->write_config($vmid, $conf);
5109
5110     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5111         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5112     }
5113 }
5114
5115 sub try_deallocate_drive {
5116     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5117
5118     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5119         my $volid = $drive->{file};
5120         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5121             my $sid = PVE::Storage::parse_volume_id($volid);
5122             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5123
5124             # check if the disk is really unused
5125             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5126                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5127             PVE::Storage::vdisk_free($storecfg, $volid);
5128             return 1;
5129         } else {
5130             # If vm is not owner of this disk remove from config
5131             return 1;
5132         }
5133     }
5134
5135     return;
5136 }
5137
5138 sub vmconfig_delete_or_detach_drive {
5139     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5140
5141     my $drive = parse_drive($opt, $conf->{$opt});
5142
5143     my $rpcenv = PVE::RPCEnvironment::get();
5144     my $authuser = $rpcenv->get_user();
5145
5146     if ($force) {
5147         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5148         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5149     } else {
5150         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5151     }
5152 }
5153
5154
5155
5156 sub vmconfig_apply_pending {
5157     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5158
5159     return if !scalar(keys %{$conf->{pending}});
5160
5161     my $add_apply_error = sub {
5162         my ($opt, $msg) = @_;
5163         my $err_msg = "unable to apply pending change $opt : $msg";
5164         $errors->{$opt} = $err_msg;
5165         warn $err_msg;
5166     };
5167
5168     # cold plug
5169
5170     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5171     foreach my $opt (sort keys %$pending_delete_hash) {
5172         my $force = $pending_delete_hash->{$opt}->{force};
5173         eval {
5174             if ($opt =~ m/^unused/) {
5175                 die "internal error";
5176             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5177                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5178             } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5179                 if($have_sdn) {
5180                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5181                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5182                     warn if $@;
5183                 }
5184             }
5185         };
5186         if (my $err = $@) {
5187             $add_apply_error->($opt, $err);
5188         } else {
5189             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5190             delete $conf->{$opt};
5191         }
5192     }
5193
5194     PVE::QemuConfig->cleanup_pending($conf);
5195
5196     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5197
5198     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5199         next if $opt eq 'delete'; # just to be sure
5200         eval {
5201             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5202                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5203             } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5204                 if($have_sdn) {
5205                     my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5206                     if ($conf->{$opt}){
5207                         my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5208
5209                         if ($old_net->{bridge} ne $new_net->{bridge} ||
5210                             $old_net->{macaddr} ne $new_net->{macaddr}) {
5211                             PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5212                         }
5213                    }
5214                    #fixme: reuse ip if mac change && same bridge
5215                    PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5216                 }
5217             }
5218         };
5219         if (my $err = $@) {
5220             $add_apply_error->($opt, $err);
5221         } else {
5222
5223             if (is_valid_drivename($opt)) {
5224                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5225                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5226             }
5227
5228             $conf->{$opt} = delete $conf->{pending}->{$opt};
5229         }
5230     }
5231
5232     # write all changes at once to avoid unnecessary i/o
5233     PVE::QemuConfig->write_config($vmid, $conf);
5234     if ($generate_cloudinit) {
5235         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5236             # After successful generation and if there were changes to be applied, update the
5237             # config to drop the {cloudinit} entry.
5238             PVE::QemuConfig->write_config($vmid, $conf);
5239         }
5240     }
5241 }
5242
5243 sub vmconfig_update_net {
5244     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5245
5246     my $newnet = parse_net($value);
5247
5248     if ($conf->{$opt}) {
5249         my $oldnet = parse_net($conf->{$opt});
5250
5251         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5252             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5253             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5254             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5255             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5256
5257             # for non online change, we try to hot-unplug
5258             die "skip\n" if !$hotplug;
5259             vm_deviceunplug($vmid, $conf, $opt);
5260
5261             if($have_sdn) {
5262                 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5263             }
5264
5265         } else {
5266
5267             die "internal error" if $opt !~ m/net(\d+)/;
5268             my $iface = "tap${vmid}i$1";
5269
5270             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5271                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5272                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5273                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5274                 PVE::Network::tap_unplug($iface);
5275
5276                 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5277                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5278                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5279                     qemu_set_link_status($vmid, $opt, 0);
5280                 }
5281
5282                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5283                     if ($have_sdn) {
5284                         PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5285                         PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5286                     }
5287                 }
5288
5289                 if ($have_sdn) {
5290                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5291                 } else {
5292                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5293                 }
5294
5295                 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5296                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5297                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5298                     qemu_set_link_status($vmid, $opt, 1);
5299                 }
5300
5301             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5302                 # Rate can be applied on its own but any change above needs to
5303                 # include the rate in tap_plug since OVS resets everything.
5304                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5305             }
5306
5307             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5308                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5309             }
5310
5311             return 1;
5312         }
5313     }
5314
5315     if ($hotplug) {
5316         if ($have_sdn) {
5317             PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5318             PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5319         }
5320         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5321     } else {
5322         die "skip\n";
5323     }
5324 }
5325
5326 sub vmconfig_update_agent {
5327     my ($conf, $opt, $value) = @_;
5328
5329     die "skip\n" if !$conf->{$opt};
5330
5331     my $hotplug_options = { fstrim_cloned_disks => 1 };
5332
5333     my $old_agent = parse_guest_agent($conf);
5334     my $agent = parse_guest_agent({$opt => $value});
5335
5336     for my $option (keys %$agent) { # added/changed options
5337         next if defined($hotplug_options->{$option});
5338         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5339     }
5340
5341     for my $option (keys %$old_agent) { # removed options
5342         next if defined($hotplug_options->{$option});
5343         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5344     }
5345
5346     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5347 }
5348
5349 sub vmconfig_update_disk {
5350     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5351
5352     my $drive = parse_drive($opt, $value);
5353
5354     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5355         my $media = $drive->{media} || 'disk';
5356         my $oldmedia = $old_drive->{media} || 'disk';
5357         die "unable to change media type\n" if $media ne $oldmedia;
5358
5359         if (!drive_is_cdrom($old_drive)) {
5360
5361             if ($drive->{file} ne $old_drive->{file}) {
5362
5363                 die "skip\n" if !$hotplug;
5364
5365                 # unplug and register as unused
5366                 vm_deviceunplug($vmid, $conf, $opt);
5367                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5368
5369             } else {
5370                 # update existing disk
5371
5372                 # skip non hotpluggable value
5373                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5374                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5375                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5376                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5377                     safe_string_ne($drive->{product}, $old_drive->{product}) ||
5378                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5379                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5380                     safe_string_ne($drive->{vendor}, $old_drive->{vendor}) ||
5381                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5382                     die "skip\n";
5383                 }
5384
5385                 # apply throttle
5386                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5387                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5388                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5389                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5390                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5391                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5392                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5393                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5394                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5395                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5396                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5397                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5398                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5399                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5400                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5401                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5402                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5403                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5404
5405                     qemu_block_set_io_throttle(
5406                         $vmid,"drive-$opt",
5407                         ($drive->{mbps} || 0)*1024*1024,
5408                         ($drive->{mbps_rd} || 0)*1024*1024,
5409                         ($drive->{mbps_wr} || 0)*1024*1024,
5410                          $drive->{iops} || 0,
5411                          $drive->{iops_rd} || 0,
5412                          $drive->{iops_wr} || 0,
5413                         ($drive->{mbps_max} || 0)*1024*1024,
5414                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5415                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5416                          $drive->{iops_max} || 0,
5417                          $drive->{iops_rd_max} || 0,
5418                          $drive->{iops_wr_max} || 0,
5419                          $drive->{bps_max_length} || 1,
5420                          $drive->{bps_rd_max_length} || 1,
5421                          $drive->{bps_wr_max_length} || 1,
5422                          $drive->{iops_max_length} || 1,
5423                          $drive->{iops_rd_max_length} || 1,
5424                          $drive->{iops_wr_max_length} || 1,
5425                     );
5426
5427                 }
5428
5429                 return 1;
5430             }
5431
5432         } else { # cdrom
5433
5434             if ($drive->{file} eq 'none') {
5435                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5436                 if (drive_is_cloudinit($old_drive)) {
5437                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5438                 }
5439             } else {
5440                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5441
5442                 # force eject if locked
5443                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5444
5445                 if ($path) {
5446                     mon_cmd($vmid, "blockdev-change-medium",
5447                         id => "$opt", filename => "$path");
5448                 }
5449             }
5450
5451             return 1;
5452         }
5453     }
5454
5455     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5456     # hotplug new disks
5457     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5458     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5459 }
5460
5461 sub vmconfig_update_cloudinit_drive {
5462     my ($storecfg, $conf, $vmid) = @_;
5463
5464     my $cloudinit_ds = undef;
5465     my $cloudinit_drive = undef;
5466
5467     PVE::QemuConfig->foreach_volume($conf, sub {
5468         my ($ds, $drive) = @_;
5469         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5470             $cloudinit_ds = $ds;
5471             $cloudinit_drive = $drive;
5472         }
5473     });
5474
5475     return if !$cloudinit_drive;
5476
5477     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5478         PVE::QemuConfig->write_config($vmid, $conf);
5479     }
5480
5481     my $running = PVE::QemuServer::check_running($vmid);
5482
5483     if ($running) {
5484         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5485         if ($path) {
5486             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5487             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5488         }
5489     }
5490 }
5491
5492 # called in locked context by incoming migration
5493 sub vm_migrate_get_nbd_disks {
5494     my ($storecfg, $conf, $replicated_volumes) = @_;
5495
5496     my $local_volumes = {};
5497     PVE::QemuConfig->foreach_volume($conf, sub {
5498         my ($ds, $drive) = @_;
5499
5500         return if drive_is_cdrom($drive);
5501         return if $ds eq 'tpmstate0';
5502
5503         my $volid = $drive->{file};
5504
5505         return if !$volid;
5506
5507         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5508
5509         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5510         return if $scfg->{shared};
5511
5512         my $format = qemu_img_format($scfg, $volname);
5513
5514         # replicated disks re-use existing state via bitmap
5515         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5516         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5517     });
5518     return $local_volumes;
5519 }
5520
5521 # called in locked context by incoming migration
5522 sub vm_migrate_alloc_nbd_disks {
5523     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5524
5525     my $nbd = {};
5526     foreach my $opt (sort keys %$source_volumes) {
5527         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5528
5529         if ($use_existing) {
5530             $nbd->{$opt}->{drivestr} = print_drive($drive);
5531             $nbd->{$opt}->{volid} = $volid;
5532             $nbd->{$opt}->{replicated} = 1;
5533             next;
5534         }
5535
5536         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5537
5538         # order of precedence, filtered by whether storage supports it:
5539         # 1. explicit requested format
5540         # 2. default format of storage
5541         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5542         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5543
5544         my $size = $drive->{size} / 1024;
5545         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5546         my $newdrive = $drive;
5547         $newdrive->{format} = $format;
5548         $newdrive->{file} = $newvolid;
5549         my $drivestr = print_drive($newdrive);
5550         $nbd->{$opt}->{drivestr} = $drivestr;
5551         $nbd->{$opt}->{volid} = $newvolid;
5552     }
5553
5554     return $nbd;
5555 }
5556
5557 # see vm_start_nolock for parameters, additionally:
5558 # migrate_opts:
5559 #   storagemap = parsed storage map for allocating NBD disks
5560 sub vm_start {
5561     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5562
5563     return PVE::QemuConfig->lock_config($vmid, sub {
5564         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5565
5566         die "you can't start a vm if it's a template\n"
5567             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5568
5569         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5570         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5571
5572         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5573
5574         if ($has_backup_lock && $running) {
5575             # a backup is currently running, attempt to start the guest in the
5576             # existing QEMU instance
5577             return vm_resume($vmid);
5578         }
5579
5580         PVE::QemuConfig->check_lock($conf)
5581             if !($params->{skiplock} || $has_suspended_lock);
5582
5583         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5584
5585         die "VM $vmid already running\n" if $running;
5586
5587         if (my $storagemap = $migrate_opts->{storagemap}) {
5588             my $replicated = $migrate_opts->{replicated_volumes};
5589             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5590             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5591
5592             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5593                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5594             }
5595         }
5596
5597         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5598     });
5599 }
5600
5601
5602 # params:
5603 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5604 #   skiplock => 0/1, skip checking for config lock
5605 #   skiptemplate => 0/1, skip checking whether VM is template
5606 #   forcemachine => to force QEMU machine (rollback/migration)
5607 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5608 #   timeout => in seconds
5609 #   paused => start VM in paused state (backup)
5610 #   resume => resume from hibernation
5611 #   pbs-backing => {
5612 #      sata0 => {
5613 #         repository
5614 #         snapshot
5615 #         keyfile
5616 #         archive
5617 #      },
5618 #      virtio2 => ...
5619 #   }
5620 # migrate_opts:
5621 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5622 #   migratedfrom => source node
5623 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5624 #   network => CIDR of migration network
5625 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5626 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5627 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5628 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5629 #       contained in config
5630 sub vm_start_nolock {
5631     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5632
5633     my $statefile = $params->{statefile};
5634     my $resume = $params->{resume};
5635
5636     my $migratedfrom = $migrate_opts->{migratedfrom};
5637     my $migration_type = $migrate_opts->{type};
5638
5639     my $res = {};
5640
5641     # clean up leftover reboot request files
5642     eval { clear_reboot_request($vmid); };
5643     warn $@ if $@;
5644
5645     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5646         vmconfig_apply_pending($vmid, $conf, $storecfg);
5647         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5648     }
5649
5650     # don't regenerate the ISO if the VM is started as part of a live migration
5651     # this way we can reuse the old ISO with the correct config
5652     if (!$migratedfrom) {
5653         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5654             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5655             # $conf->{cloudinit}, so we could just not do this?
5656             # But we do it above, so for now let's be consistent.
5657             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5658         }
5659     }
5660
5661     # override offline migrated volumes, conf is out of date still
5662     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5663         for my $key (sort keys $offline_volumes->%*) {
5664             my $parsed = parse_drive($key, $conf->{$key});
5665             $parsed->{file} = $offline_volumes->{$key};
5666             $conf->{$key} = print_drive($parsed);
5667         }
5668     }
5669
5670     my $defaults = load_defaults();
5671
5672     # set environment variable useful inside network script
5673     # for remote migration the config is available on the target node!
5674     if (!$migrate_opts->{remote_node}) {
5675         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5676     }
5677
5678     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5679
5680     my $forcemachine = $params->{forcemachine};
5681     my $forcecpu = $params->{forcecpu};
5682     if ($resume) {
5683         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5684         $forcemachine = $conf->{runningmachine};
5685         $forcecpu = $conf->{runningcpu};
5686         print "Resuming suspended VM\n";
5687     }
5688
5689     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5690         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5691
5692     my $migration_ip;
5693     my $get_migration_ip = sub {
5694         my ($nodename) = @_;
5695
5696         return $migration_ip if defined($migration_ip);
5697
5698         my $cidr = $migrate_opts->{network};
5699
5700         if (!defined($cidr)) {
5701             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5702             $cidr = $dc_conf->{migration}->{network};
5703         }
5704
5705         if (defined($cidr)) {
5706             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5707
5708             die "could not get IP: no address configured on local " .
5709                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5710
5711             die "could not get IP: multiple addresses configured on local " .
5712                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5713
5714             $migration_ip = @$ips[0];
5715         }
5716
5717         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5718             if !defined($migration_ip);
5719
5720         return $migration_ip;
5721     };
5722
5723     if ($statefile) {
5724         if ($statefile eq 'tcp') {
5725             my $migrate = $res->{migrate} = { proto => 'tcp' };
5726             $migrate->{addr} = "localhost";
5727             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5728             my $nodename = nodename();
5729
5730             if (!defined($migration_type)) {
5731                 if (defined($datacenterconf->{migration}->{type})) {
5732                     $migration_type = $datacenterconf->{migration}->{type};
5733                 } else {
5734                     $migration_type = 'secure';
5735                 }
5736             }
5737
5738             if ($migration_type eq 'insecure') {
5739                 $migrate->{addr} = $get_migration_ip->($nodename);
5740                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5741             }
5742
5743             # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5744             # via QMP later
5745             push @$cmd, '-incoming', 'defer';
5746             push @$cmd, '-S';
5747
5748         } elsif ($statefile eq 'unix') {
5749             # should be default for secure migrations as a ssh TCP forward
5750             # tunnel is not deterministic reliable ready and fails regurarly
5751             # to set up in time, so use UNIX socket forwards
5752             my $migrate = $res->{migrate} = { proto => 'unix' };
5753             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5754             unlink $migrate->{addr};
5755
5756             $migrate->{uri} = "unix:$migrate->{addr}";
5757             push @$cmd, '-incoming', $migrate->{uri};
5758             push @$cmd, '-S';
5759
5760         } elsif (-e $statefile) {
5761             push @$cmd, '-loadstate', $statefile;
5762         } else {
5763             my $statepath = PVE::Storage::path($storecfg, $statefile);
5764             push @$vollist, $statefile;
5765             push @$cmd, '-loadstate', $statepath;
5766         }
5767     } elsif ($params->{paused}) {
5768         push @$cmd, '-S';
5769     }
5770
5771     my $memory = get_current_memory($conf->{memory});
5772     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5773
5774     my $pci_reserve_list = [];
5775     for my $device (values $pci_devices->%*) {
5776         next if $device->{mdev}; # we don't reserve for mdev devices
5777         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5778     }
5779
5780     # reserve all PCI IDs before actually doing anything with them
5781     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5782
5783     eval {
5784         my $uuid;
5785         for my $id (sort keys %$pci_devices) {
5786             my $d = $pci_devices->{$id};
5787             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5788
5789             my $chosen_mdev;
5790             for my $dev ($d->{ids}->@*) {
5791                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5792                 if ($d->{mdev}) {
5793                     warn $@ if $@;
5794                     $chosen_mdev = $info;
5795                     last if $chosen_mdev; # if successful, we're done
5796                 } else {
5797                     die $@ if $@;
5798                 }
5799             }
5800
5801             next if !$d->{mdev};
5802             die "could not create mediated device\n" if !defined($chosen_mdev);
5803
5804             # nvidia grid needs the uuid of the mdev as qemu parameter
5805             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5806                 if (defined($conf->{smbios1})) {
5807                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5808                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5809                 }
5810                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5811             }
5812         }
5813         push @$cmd, '-uuid', $uuid if defined($uuid);
5814     };
5815     if (my $err = $@) {
5816         eval { cleanup_pci_devices($vmid, $conf) };
5817         warn $@ if $@;
5818         die $err;
5819     }
5820
5821     PVE::Storage::activate_volumes($storecfg, $vollist);
5822
5823
5824     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5825     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5826     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5827     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5828     # timeout should be more than enough here...
5829     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5830
5831     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5832
5833     my %run_params = (
5834         timeout => $statefile ? undef : $start_timeout,
5835         umask => 0077,
5836         noerr => 1,
5837     );
5838
5839     # when migrating, prefix QEMU output so other side can pick up any
5840     # errors that might occur and show the user
5841     if ($migratedfrom) {
5842         $run_params{quiet} = 1;
5843         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5844     }
5845
5846     my %systemd_properties = (
5847         Slice => 'qemu.slice',
5848         KillMode => 'process',
5849         SendSIGKILL => 0,
5850         TimeoutStopUSec => ULONG_MAX, # infinity
5851     );
5852
5853     if (PVE::CGroup::cgroup_mode() == 2) {
5854         $systemd_properties{CPUWeight} = $cpuunits;
5855     } else {
5856         $systemd_properties{CPUShares} = $cpuunits;
5857     }
5858
5859     if (my $cpulimit = $conf->{cpulimit}) {
5860         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5861     }
5862     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5863
5864     my $run_qemu = sub {
5865         PVE::Tools::run_fork sub {
5866             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5867
5868             my $tpmpid;
5869             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5870                 # start the TPM emulator so QEMU can connect on start
5871                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5872             }
5873
5874             my $exitcode = run_command($cmd, %run_params);
5875             if ($exitcode) {
5876                 if ($tpmpid) {
5877                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5878                     kill 'TERM', $tpmpid;
5879                 }
5880                 die "QEMU exited with code $exitcode\n";
5881             }
5882         };
5883     };
5884
5885     if ($conf->{hugepages}) {
5886
5887         my $code = sub {
5888             my $hotplug_features =
5889                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5890             my $hugepages_topology =
5891                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5892
5893             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5894
5895             PVE::QemuServer::Memory::hugepages_mount();
5896             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5897
5898             eval { $run_qemu->() };
5899             if (my $err = $@) {
5900                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5901                     if !$conf->{keephugepages};
5902                 die $err;
5903             }
5904
5905             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5906                 if !$conf->{keephugepages};
5907         };
5908         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5909
5910     } else {
5911         eval { $run_qemu->() };
5912     }
5913
5914     if (my $err = $@) {
5915         # deactivate volumes if start fails
5916         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5917         warn $@ if $@;
5918         eval { cleanup_pci_devices($vmid, $conf) };
5919         warn $@ if $@;
5920
5921         die "start failed: $err";
5922     }
5923
5924     # re-reserve all PCI IDs now that we can know the actual VM PID
5925     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5926     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5927     warn $@ if $@;
5928
5929     if (defined(my $migrate = $res->{migrate})) {
5930         if ($migrate->{proto} eq 'tcp') {
5931             my $nodename = nodename();
5932             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5933             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5934             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5935             mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
5936         }
5937         print "migration listens on $migrate->{uri}\n";
5938     } elsif ($statefile) {
5939         eval { mon_cmd($vmid, "cont"); };
5940         warn $@ if $@;
5941     }
5942
5943     #start nbd server for storage migration
5944     if (my $nbd = $migrate_opts->{nbd}) {
5945         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5946
5947         my $migrate_storage_uri;
5948         # nbd_protocol_version > 0 for unix socket support
5949         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5950             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5951             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5952             $migrate_storage_uri = "nbd:unix:$socket_path";
5953             $res->{migrate}->{unix_sockets} = [$socket_path];
5954         } else {
5955             my $nodename = nodename();
5956             my $localip = $get_migration_ip->($nodename);
5957             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5958             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5959
5960             mon_cmd($vmid, "nbd-server-start", addr => {
5961                 type => 'inet',
5962                 data => {
5963                     host => "${localip}",
5964                     port => "${storage_migrate_port}",
5965                 },
5966             });
5967             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5968             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5969         }
5970
5971         my $block_info = mon_cmd($vmid, "query-block");
5972         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5973
5974         foreach my $opt (sort keys %$nbd) {
5975             my $drivestr = $nbd->{$opt}->{drivestr};
5976             my $volid = $nbd->{$opt}->{volid};
5977
5978             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5979
5980             mon_cmd(
5981                 $vmid,
5982                 "block-export-add",
5983                 id => "drive-$opt",
5984                 'node-name' => $block_node,
5985                 writable => JSON::true,
5986                 type => "nbd",
5987                 name => "drive-$opt", # NBD export name
5988             );
5989
5990             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5991             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5992             print "re-using replicated volume: $opt - $volid\n"
5993                 if $nbd->{$opt}->{replicated};
5994
5995             $res->{drives}->{$opt} = $nbd->{$opt};
5996             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5997         }
5998     }
5999
6000     if ($migratedfrom) {
6001         eval {
6002             set_migration_caps($vmid);
6003         };
6004         warn $@ if $@;
6005
6006         if ($spice_port) {
6007             print "spice listens on port $spice_port\n";
6008             $res->{spice_port} = $spice_port;
6009             if ($migrate_opts->{spice_ticket}) {
6010                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6011                     $migrate_opts->{spice_ticket});
6012                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6013             }
6014         }
6015
6016     } else {
6017         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6018             if !$statefile && $conf->{balloon};
6019
6020         foreach my $opt (keys %$conf) {
6021             next if $opt !~  m/^net\d+$/;
6022             my $nicconf = parse_net($conf->{$opt});
6023             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6024         }
6025         add_nets_bridge_fdb($conf, $vmid);
6026     }
6027
6028    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6029         eval {
6030             mon_cmd(
6031                 $vmid,
6032                 'qom-set',
6033                 path => "machine/peripheral/balloon0",
6034                 property => "guest-stats-polling-interval",
6035                 value => 2
6036             );
6037         };
6038         log_warn("could not set polling interval for ballooning - $@") if $@;
6039     }
6040
6041     if ($resume) {
6042         print "Resumed VM, removing state\n";
6043         if (my $vmstate = $conf->{vmstate}) {
6044             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6045             PVE::Storage::vdisk_free($storecfg, $vmstate);
6046         }
6047         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6048         PVE::QemuConfig->write_config($vmid, $conf);
6049     }
6050
6051     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6052
6053     my ($current_machine, $is_deprecated) =
6054         PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6055     if ($is_deprecated) {
6056         log_warn(
6057             "current machine version '$current_machine' is deprecated - see the documentation and ".
6058             "change to a newer one",
6059         );
6060     }
6061
6062     return $res;
6063 }
6064
6065 sub vm_commandline {
6066     my ($storecfg, $vmid, $snapname) = @_;
6067
6068     my $conf = PVE::QemuConfig->load_config($vmid);
6069
6070     my ($forcemachine, $forcecpu);
6071     if ($snapname) {
6072         my $snapshot = $conf->{snapshots}->{$snapname};
6073         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6074
6075         # check for machine or CPU overrides in snapshot
6076         $forcemachine = $snapshot->{runningmachine};
6077         $forcecpu = $snapshot->{runningcpu};
6078
6079         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6080
6081         $conf = $snapshot;
6082     }
6083
6084     my $defaults = load_defaults();
6085
6086     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6087
6088     return PVE::Tools::cmd2string($cmd);
6089 }
6090
6091 sub vm_reset {
6092     my ($vmid, $skiplock) = @_;
6093
6094     PVE::QemuConfig->lock_config($vmid, sub {
6095
6096         my $conf = PVE::QemuConfig->load_config($vmid);
6097
6098         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6099
6100         mon_cmd($vmid, "system_reset");
6101     });
6102 }
6103
6104 sub get_vm_volumes {
6105     my ($conf) = @_;
6106
6107     my $vollist = [];
6108     foreach_volid($conf, sub {
6109         my ($volid, $attr) = @_;
6110
6111         return if $volid =~ m|^/|;
6112
6113         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6114         return if !$sid;
6115
6116         push @$vollist, $volid;
6117     });
6118
6119     return $vollist;
6120 }
6121
6122 sub cleanup_pci_devices {
6123     my ($vmid, $conf) = @_;
6124
6125     foreach my $key (keys %$conf) {
6126         next if $key !~ m/^hostpci(\d+)$/;
6127         my $hostpciindex = $1;
6128         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6129         my $d = parse_hostpci($conf->{$key});
6130         if ($d->{mdev}) {
6131             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6132             # don't want to break ABI just for this two liner
6133             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6134
6135             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6136             # out when we do it first. so wait for 10 seconds and then try it
6137             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/) {
6138                 sleep 10;
6139             }
6140
6141             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6142         }
6143     }
6144     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6145 }
6146
6147 sub vm_stop_cleanup {
6148     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6149
6150     eval {
6151
6152         if (!$keepActive) {
6153             my $vollist = get_vm_volumes($conf);
6154             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6155
6156             if (my $tpmdrive = $conf->{tpmstate0}) {
6157                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6158                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6159                 if ($storeid) {
6160                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6161                 }
6162             }
6163         }
6164
6165         foreach my $ext (qw(mon qmp pid vnc qga)) {
6166             unlink "/var/run/qemu-server/${vmid}.$ext";
6167         }
6168
6169         if ($conf->{ivshmem}) {
6170             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6171             # just delete it for now, VMs which have this already open do not
6172             # are affected, but new VMs will get a separated one. If this
6173             # becomes an issue we either add some sort of ref-counting or just
6174             # add a "don't delete on stop" flag to the ivshmem format.
6175             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6176         }
6177
6178         cleanup_pci_devices($vmid, $conf);
6179
6180         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6181     };
6182     warn $@ if $@; # avoid errors - just warn
6183 }
6184
6185 # call only in locked context
6186 sub _do_vm_stop {
6187     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6188
6189     my $pid = check_running($vmid, $nocheck);
6190     return if !$pid;
6191
6192     my $conf;
6193     if (!$nocheck) {
6194         $conf = PVE::QemuConfig->load_config($vmid);
6195         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6196         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6197             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6198             $timeout = $opts->{down} if $opts->{down};
6199         }
6200         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6201     }
6202
6203     eval {
6204         if ($shutdown) {
6205             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6206                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6207             } else {
6208                 mon_cmd($vmid, "system_powerdown");
6209             }
6210         } else {
6211             mon_cmd($vmid, "quit");
6212         }
6213     };
6214     my $err = $@;
6215
6216     if (!$err) {
6217         $timeout = 60 if !defined($timeout);
6218
6219         my $count = 0;
6220         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6221             $count++;
6222             sleep 1;
6223         }
6224
6225         if ($count >= $timeout) {
6226             if ($force) {
6227                 warn "VM still running - terminating now with SIGTERM\n";
6228                 kill 15, $pid;
6229             } else {
6230                 die "VM quit/powerdown failed - got timeout\n";
6231             }
6232         } else {
6233             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6234             return;
6235         }
6236     } else {
6237        if (!check_running($vmid, $nocheck)) {
6238             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6239             return;
6240        }
6241        if ($force) {
6242             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6243             kill 15, $pid;
6244         } else {
6245             die "VM quit/powerdown failed\n";
6246         }
6247     }
6248
6249     # wait again
6250     $timeout = 10;
6251
6252     my $count = 0;
6253     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6254         $count++;
6255         sleep 1;
6256     }
6257
6258     if ($count >= $timeout) {
6259         warn "VM still running - terminating now with SIGKILL\n";
6260         kill 9, $pid;
6261         sleep 1;
6262     }
6263
6264     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6265 }
6266
6267 # Note: use $nocheck to skip tests if VM configuration file exists.
6268 # We need that when migration VMs to other nodes (files already moved)
6269 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6270 sub vm_stop {
6271     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6272
6273     $force = 1 if !defined($force) && !$shutdown;
6274
6275     if ($migratedfrom){
6276         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6277         kill 15, $pid if $pid;
6278         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6279         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6280         return;
6281     }
6282
6283     PVE::QemuConfig->lock_config($vmid, sub {
6284         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6285    });
6286 }
6287
6288 sub vm_reboot {
6289     my ($vmid, $timeout) = @_;
6290
6291     PVE::QemuConfig->lock_config($vmid, sub {
6292         eval {
6293
6294             # only reboot if running, as qmeventd starts it again on a stop event
6295             return if !check_running($vmid);
6296
6297             create_reboot_request($vmid);
6298
6299             my $storecfg = PVE::Storage::config();
6300             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6301
6302         };
6303         if (my $err = $@) {
6304             # avoid that the next normal shutdown will be confused for a reboot
6305             clear_reboot_request($vmid);
6306             die $err;
6307         }
6308    });
6309 }
6310
6311 # note: if using the statestorage parameter, the caller has to check privileges
6312 sub vm_suspend {
6313     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6314
6315     my $conf;
6316     my $path;
6317     my $storecfg;
6318     my $vmstate;
6319
6320     PVE::QemuConfig->lock_config($vmid, sub {
6321
6322         $conf = PVE::QemuConfig->load_config($vmid);
6323
6324         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6325         PVE::QemuConfig->check_lock($conf)
6326             if !($skiplock || $is_backing_up);
6327
6328         die "cannot suspend to disk during backup\n"
6329             if $is_backing_up && $includestate;
6330
6331         if ($includestate) {
6332             $conf->{lock} = 'suspending';
6333             my $date = strftime("%Y-%m-%d", localtime(time()));
6334             $storecfg = PVE::Storage::config();
6335             if (!$statestorage) {
6336                 $statestorage = find_vmstate_storage($conf, $storecfg);
6337                 # check permissions for the storage
6338                 my $rpcenv = PVE::RPCEnvironment::get();
6339                 if ($rpcenv->{type} ne 'cli') {
6340                     my $authuser = $rpcenv->get_user();
6341                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6342                 }
6343             }
6344
6345
6346             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6347                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6348             $path = PVE::Storage::path($storecfg, $vmstate);
6349             PVE::QemuConfig->write_config($vmid, $conf);
6350         } else {
6351             mon_cmd($vmid, "stop");
6352         }
6353     });
6354
6355     if ($includestate) {
6356         # save vm state
6357         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6358
6359         eval {
6360             set_migration_caps($vmid, 1);
6361             mon_cmd($vmid, "savevm-start", statefile => $path);
6362             for(;;) {
6363                 my $state = mon_cmd($vmid, "query-savevm");
6364                 if (!$state->{status}) {
6365                     die "savevm not active\n";
6366                 } elsif ($state->{status} eq 'active') {
6367                     sleep(1);
6368                     next;
6369                 } elsif ($state->{status} eq 'completed') {
6370                     print "State saved, quitting\n";
6371                     last;
6372                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6373                     die "query-savevm failed with error '$state->{error}'\n"
6374                 } else {
6375                     die "query-savevm returned status '$state->{status}'\n";
6376                 }
6377             }
6378         };
6379         my $err = $@;
6380
6381         PVE::QemuConfig->lock_config($vmid, sub {
6382             $conf = PVE::QemuConfig->load_config($vmid);
6383             if ($err) {
6384                 # cleanup, but leave suspending lock, to indicate something went wrong
6385                 eval {
6386                     mon_cmd($vmid, "savevm-end");
6387                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6388                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6389                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6390                     PVE::QemuConfig->write_config($vmid, $conf);
6391                 };
6392                 warn $@ if $@;
6393                 die $err;
6394             }
6395
6396             die "lock changed unexpectedly\n"
6397                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6398
6399             mon_cmd($vmid, "quit");
6400             $conf->{lock} = 'suspended';
6401             PVE::QemuConfig->write_config($vmid, $conf);
6402         });
6403     }
6404 }
6405
6406 # $nocheck is set when called as part of a migration - in this context the
6407 # location of the config file (source or target node) is not deterministic,
6408 # since migration cannot wait for pmxcfs to process the rename
6409 sub vm_resume {
6410     my ($vmid, $skiplock, $nocheck) = @_;
6411
6412     PVE::QemuConfig->lock_config($vmid, sub {
6413         my $res = mon_cmd($vmid, 'query-status');
6414         my $resume_cmd = 'cont';
6415         my $reset = 0;
6416         my $conf;
6417         if ($nocheck) {
6418             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6419             if ($@) {
6420                 my $vmlist = PVE::Cluster::get_vmlist();
6421                 if (exists($vmlist->{ids}->{$vmid})) {
6422                     my $node = $vmlist->{ids}->{$vmid}->{node};
6423                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6424                 }
6425                 if (!$conf) {
6426                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6427                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6428                 }
6429             }
6430         } else {
6431             $conf = PVE::QemuConfig->load_config($vmid);
6432         }
6433
6434         if ($res->{status}) {
6435             return if $res->{status} eq 'running'; # job done, go home
6436             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6437             $reset = 1 if $res->{status} eq 'shutdown';
6438         }
6439
6440         if (!$nocheck) {
6441             PVE::QemuConfig->check_lock($conf)
6442                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6443         }
6444
6445         if ($reset) {
6446             # required if a VM shuts down during a backup and we get a resume
6447             # request before the backup finishes for example
6448             mon_cmd($vmid, "system_reset");
6449         }
6450
6451         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6452
6453         mon_cmd($vmid, $resume_cmd);
6454     });
6455 }
6456
6457 sub vm_sendkey {
6458     my ($vmid, $skiplock, $key) = @_;
6459
6460     PVE::QemuConfig->lock_config($vmid, sub {
6461
6462         my $conf = PVE::QemuConfig->load_config($vmid);
6463
6464         # there is no qmp command, so we use the human monitor command
6465         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6466         die $res if $res ne '';
6467     });
6468 }
6469
6470 sub check_bridge_access {
6471     my ($rpcenv, $authuser, $conf) = @_;
6472
6473     return 1 if $authuser eq 'root@pam';
6474
6475     for my $opt (sort keys $conf->%*) {
6476         next if $opt !~ m/^net\d+$/;
6477         my $net = parse_net($conf->{$opt});
6478         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6479         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6480     }
6481     return 1;
6482 };
6483
6484 sub check_mapping_access {
6485     my ($rpcenv, $user, $conf) = @_;
6486
6487     for my $opt (keys $conf->%*) {
6488         if ($opt =~ m/^usb\d+$/) {
6489             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6490             if (my $host = $device->{host}) {
6491                 die "only root can set '$opt' config for real devices\n"
6492                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6493             } elsif ($device->{mapping}) {
6494                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6495             } else {
6496                 die "either 'host' or 'mapping' must be set.\n";
6497             }
6498         } elsif ($opt =~ m/^hostpci\d+$/) {
6499             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6500             if ($device->{host}) {
6501                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6502             } elsif ($device->{mapping}) {
6503                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6504             } else {
6505                 die "either 'host' or 'mapping' must be set.\n";
6506             }
6507        }
6508    }
6509 };
6510
6511 sub check_restore_permissions {
6512     my ($rpcenv, $user, $conf) = @_;
6513
6514     check_bridge_access($rpcenv, $user, $conf);
6515     check_mapping_access($rpcenv, $user, $conf);
6516 }
6517 # vzdump restore implementaion
6518
6519 sub tar_archive_read_firstfile {
6520     my $archive = shift;
6521
6522     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6523
6524     # try to detect archive type first
6525     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6526         die "unable to open file '$archive'\n";
6527     my $firstfile = <$fh>;
6528     kill 15, $pid;
6529     close $fh;
6530
6531     die "ERROR: archive contaions no data\n" if !$firstfile;
6532     chomp $firstfile;
6533
6534     return $firstfile;
6535 }
6536
6537 sub tar_restore_cleanup {
6538     my ($storecfg, $statfile) = @_;
6539
6540     print STDERR "starting cleanup\n";
6541
6542     if (my $fd = IO::File->new($statfile, "r")) {
6543         while (defined(my $line = <$fd>)) {
6544             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6545                 my $volid = $2;
6546                 eval {
6547                     if ($volid =~ m|^/|) {
6548                         unlink $volid || die 'unlink failed\n';
6549                     } else {
6550                         PVE::Storage::vdisk_free($storecfg, $volid);
6551                     }
6552                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6553                 };
6554                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6555             } else {
6556                 print STDERR "unable to parse line in statfile - $line";
6557             }
6558         }
6559         $fd->close();
6560     }
6561 }
6562
6563 sub restore_file_archive {
6564     my ($archive, $vmid, $user, $opts) = @_;
6565
6566     return restore_vma_archive($archive, $vmid, $user, $opts)
6567         if $archive eq '-';
6568
6569     my $info = PVE::Storage::archive_info($archive);
6570     my $format = $opts->{format} // $info->{format};
6571     my $comp = $info->{compression};
6572
6573     # try to detect archive format
6574     if ($format eq 'tar') {
6575         return restore_tar_archive($archive, $vmid, $user, $opts);
6576     } else {
6577         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6578     }
6579 }
6580
6581 # hepler to remove disks that will not be used after restore
6582 my $restore_cleanup_oldconf = sub {
6583     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6584
6585     my $kept_disks = {};
6586
6587     PVE::QemuConfig->foreach_volume($oldconf, sub {
6588         my ($ds, $drive) = @_;
6589
6590         return if drive_is_cdrom($drive, 1);
6591
6592         my $volid = $drive->{file};
6593         return if !$volid || $volid =~ m|^/|;
6594
6595         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6596         return if !$path || !$owner || ($owner != $vmid);
6597
6598         # Note: only delete disk we want to restore
6599         # other volumes will become unused
6600         if ($virtdev_hash->{$ds}) {
6601             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6602             if (my $err = $@) {
6603                 warn $err;
6604             }
6605         } else {
6606             $kept_disks->{$volid} = 1;
6607         }
6608     });
6609
6610     # after the restore we have no snapshots anymore
6611     for my $snapname (keys $oldconf->{snapshots}->%*) {
6612         my $snap = $oldconf->{snapshots}->{$snapname};
6613         if ($snap->{vmstate}) {
6614             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6615             if (my $err = $@) {
6616                 warn $err;
6617             }
6618         }
6619
6620         for my $volid (keys $kept_disks->%*) {
6621             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6622             warn $@ if $@;
6623         }
6624     }
6625 };
6626
6627 # Helper to parse vzdump backup device hints
6628 #
6629 # $rpcenv: Environment, used to ckeck storage permissions
6630 # $user: User ID, to check storage permissions
6631 # $storecfg: Storage configuration
6632 # $fh: the file handle for reading the configuration
6633 # $devinfo: should contain device sizes for all backu-up'ed devices
6634 # $options: backup options (pool, default storage)
6635 #
6636 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6637 my $parse_backup_hints = sub {
6638     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6639
6640     my $check_storage = sub { # assert if an image can be allocate
6641         my ($storeid, $scfg) = @_;
6642         die "Content type 'images' is not available on storage '$storeid'\n"
6643             if !$scfg->{content}->{images};
6644         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6645             if $user ne 'root@pam';
6646     };
6647
6648     my $virtdev_hash = {};
6649     while (defined(my $line = <$fh>)) {
6650         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6651             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6652             die "archive does not contain data for drive '$virtdev'\n"
6653                 if !$devinfo->{$devname};
6654
6655             if (defined($options->{storage})) {
6656                 $storeid = $options->{storage} || 'local';
6657             } elsif (!$storeid) {
6658                 $storeid = 'local';
6659             }
6660             $format = 'raw' if !$format;
6661             $devinfo->{$devname}->{devname} = $devname;
6662             $devinfo->{$devname}->{virtdev} = $virtdev;
6663             $devinfo->{$devname}->{format} = $format;
6664             $devinfo->{$devname}->{storeid} = $storeid;
6665
6666             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6667             $check_storage->($storeid, $scfg); # permission and content type check
6668
6669             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6670         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6671             my $virtdev = $1;
6672             my $drive = parse_drive($virtdev, $2);
6673
6674             if (drive_is_cloudinit($drive)) {
6675                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6676                 $storeid = $options->{storage} if defined ($options->{storage});
6677                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6678                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6679
6680                 $check_storage->($storeid, $scfg); # permission and content type check
6681
6682                 $virtdev_hash->{$virtdev} = {
6683                     format => $format,
6684                     storeid => $storeid,
6685                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6686                     is_cloudinit => 1,
6687                 };
6688             }
6689         }
6690     }
6691
6692     return $virtdev_hash;
6693 };
6694
6695 # Helper to allocate and activate all volumes required for a restore
6696 #
6697 # $storecfg: Storage configuration
6698 # $virtdev_hash: as returned by parse_backup_hints()
6699 #
6700 # Returns: { $virtdev => $volid }
6701 my $restore_allocate_devices = sub {
6702     my ($storecfg, $virtdev_hash, $vmid) = @_;
6703
6704     my $map = {};
6705     foreach my $virtdev (sort keys %$virtdev_hash) {
6706         my $d = $virtdev_hash->{$virtdev};
6707         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6708         my $storeid = $d->{storeid};
6709         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6710
6711         # test if requested format is supported
6712         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6713         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6714         $d->{format} = $defFormat if !$supported;
6715
6716         my $name;
6717         if ($d->{is_cloudinit}) {
6718             $name = "vm-$vmid-cloudinit";
6719             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6720             if ($scfg->{path}) {
6721                 $name .= ".$d->{format}";
6722             }
6723         }
6724
6725         my $volid = PVE::Storage::vdisk_alloc(
6726             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6727
6728         print STDERR "new volume ID is '$volid'\n";
6729         $d->{volid} = $volid;
6730
6731         PVE::Storage::activate_volumes($storecfg, [$volid]);
6732
6733         $map->{$virtdev} = $volid;
6734     }
6735
6736     return $map;
6737 };
6738
6739 sub restore_update_config_line {
6740     my ($cookie, $map, $line, $unique) = @_;
6741
6742     return '' if $line =~ m/^\#qmdump\#/;
6743     return '' if $line =~ m/^\#vzdump\#/;
6744     return '' if $line =~ m/^lock:/;
6745     return '' if $line =~ m/^unused\d+:/;
6746     return '' if $line =~ m/^parent:/;
6747
6748     my $res = '';
6749
6750     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6751     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6752         # try to convert old 1.X settings
6753         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6754         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6755             my ($model, $macaddr) = split(/\=/, $devconfig);
6756             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6757             my $net = {
6758                 model => $model,
6759                 bridge => "vmbr$ind",
6760                 macaddr => $macaddr,
6761             };
6762             my $netstr = print_net($net);
6763
6764             $res .= "net$cookie->{netcount}: $netstr\n";
6765             $cookie->{netcount}++;
6766         }
6767     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6768         my ($id, $netstr) = ($1, $2);
6769         my $net = parse_net($netstr);
6770         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6771         $netstr = print_net($net);
6772         $res .= "$id: $netstr\n";
6773     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6774         my $virtdev = $1;
6775         my $value = $3;
6776         my $di = parse_drive($virtdev, $value);
6777         if (defined($di->{backup}) && !$di->{backup}) {
6778             $res .= "#$line";
6779         } elsif ($map->{$virtdev}) {
6780             delete $di->{format}; # format can change on restore
6781             $di->{file} = $map->{$virtdev};
6782             $value = print_drive($di);
6783             $res .= "$virtdev: $value\n";
6784         } else {
6785             $res .= $line;
6786         }
6787     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6788         my $vmgenid = $1;
6789         if ($vmgenid ne '0') {
6790             # always generate a new vmgenid if there was a valid one setup
6791             $vmgenid = generate_uuid();
6792         }
6793         $res .= "vmgenid: $vmgenid\n";
6794     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6795         my ($uuid, $uuid_str);
6796         UUID::generate($uuid);
6797         UUID::unparse($uuid, $uuid_str);
6798         my $smbios1 = parse_smbios1($2);
6799         $smbios1->{uuid} = $uuid_str;
6800         $res .= $1.print_smbios1($smbios1)."\n";
6801     } else {
6802         $res .= $line;
6803     }
6804
6805     return $res;
6806 }
6807
6808 my $restore_deactivate_volumes = sub {
6809     my ($storecfg, $virtdev_hash) = @_;
6810
6811     my $vollist = [];
6812     for my $dev (values $virtdev_hash->%*) {
6813         push $vollist->@*, $dev->{volid} if $dev->{volid};
6814     }
6815
6816     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6817     print STDERR $@ if $@;
6818 };
6819
6820 my $restore_destroy_volumes = sub {
6821     my ($storecfg, $virtdev_hash) = @_;
6822
6823     for my $dev (values $virtdev_hash->%*) {
6824         my $volid = $dev->{volid} or next;
6825         eval {
6826             PVE::Storage::vdisk_free($storecfg, $volid);
6827             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6828         };
6829         print STDERR "unable to cleanup '$volid' - $@" if $@;
6830     }
6831 };
6832
6833 sub restore_merge_config {
6834     my ($filename, $backup_conf_raw, $override_conf) = @_;
6835
6836     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6837     for my $key (keys $override_conf->%*) {
6838         $backup_conf->{$key} = $override_conf->{$key};
6839     }
6840
6841     return $backup_conf;
6842 }
6843
6844 sub scan_volids {
6845     my ($cfg, $vmid) = @_;
6846
6847     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6848
6849     my $volid_hash = {};
6850     foreach my $storeid (keys %$info) {
6851         foreach my $item (@{$info->{$storeid}}) {
6852             next if !($item->{volid} && $item->{size});
6853             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6854             $volid_hash->{$item->{volid}} = $item;
6855         }
6856     }
6857
6858     return $volid_hash;
6859 }
6860
6861 sub update_disk_config {
6862     my ($vmid, $conf, $volid_hash) = @_;
6863
6864     my $changes;
6865     my $prefix = "VM $vmid";
6866
6867     # used and unused disks
6868     my $referenced = {};
6869
6870     # Note: it is allowed to define multiple storages with same path (alias), so
6871     # we need to check both 'volid' and real 'path' (two different volid can point
6872     # to the same path).
6873
6874     my $referencedpath = {};
6875
6876     # update size info
6877     PVE::QemuConfig->foreach_volume($conf, sub {
6878         my ($opt, $drive) = @_;
6879
6880         my $volid = $drive->{file};
6881         return if !$volid;
6882         my $volume = $volid_hash->{$volid};
6883
6884         # mark volid as "in-use" for next step
6885         $referenced->{$volid} = 1;
6886         if ($volume && (my $path = $volume->{path})) {
6887             $referencedpath->{$path} = 1;
6888         }
6889
6890         return if drive_is_cdrom($drive);
6891         return if !$volume;
6892
6893         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6894         if (defined($updated)) {
6895             $changes = 1;
6896             $conf->{$opt} = print_drive($updated);
6897             print "$prefix ($opt): $msg\n";
6898         }
6899     });
6900
6901     # remove 'unusedX' entry if volume is used
6902     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6903         my ($opt, $drive) = @_;
6904
6905         my $volid = $drive->{file};
6906         return if !$volid;
6907
6908         my $path;
6909         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6910         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6911             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6912             $changes = 1;
6913             delete $conf->{$opt};
6914         }
6915
6916         $referenced->{$volid} = 1;
6917         $referencedpath->{$path} = 1 if $path;
6918     });
6919
6920     foreach my $volid (sort keys %$volid_hash) {
6921         next if $volid =~ m/vm-$vmid-state-/;
6922         next if $referenced->{$volid};
6923         my $path = $volid_hash->{$volid}->{path};
6924         next if !$path; # just to be sure
6925         next if $referencedpath->{$path};
6926         $changes = 1;
6927         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6928         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6929         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6930     }
6931
6932     return $changes;
6933 }
6934
6935 sub rescan {
6936     my ($vmid, $nolock, $dryrun) = @_;
6937
6938     my $cfg = PVE::Storage::config();
6939
6940     print "rescan volumes...\n";
6941     my $volid_hash = scan_volids($cfg, $vmid);
6942
6943     my $updatefn =  sub {
6944         my ($vmid) = @_;
6945
6946         my $conf = PVE::QemuConfig->load_config($vmid);
6947
6948         PVE::QemuConfig->check_lock($conf);
6949
6950         my $vm_volids = {};
6951         foreach my $volid (keys %$volid_hash) {
6952             my $info = $volid_hash->{$volid};
6953             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6954         }
6955
6956         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6957
6958         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6959     };
6960
6961     if (defined($vmid)) {
6962         if ($nolock) {
6963             &$updatefn($vmid);
6964         } else {
6965             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6966         }
6967     } else {
6968         my $vmlist = config_list();
6969         foreach my $vmid (keys %$vmlist) {
6970             if ($nolock) {
6971                 &$updatefn($vmid);
6972             } else {
6973                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6974             }
6975         }
6976     }
6977 }
6978
6979 sub restore_proxmox_backup_archive {
6980     my ($archive, $vmid, $user, $options) = @_;
6981
6982     my $storecfg = PVE::Storage::config();
6983
6984     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6985     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6986
6987     my $fingerprint = $scfg->{fingerprint};
6988     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6989
6990     my $repo = PVE::PBSClient::get_repository($scfg);
6991     my $namespace = $scfg->{namespace};
6992
6993     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6994     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6995     local $ENV{PBS_PASSWORD} = $password;
6996     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6997
6998     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6999         PVE::Storage::parse_volname($storecfg, $archive);
7000
7001     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7002
7003     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7004
7005     my $tmpdir = "/var/tmp/vzdumptmp$$";
7006     rmtree $tmpdir;
7007     mkpath $tmpdir;
7008
7009     my $conffile = PVE::QemuConfig->config_file($vmid);
7010      # disable interrupts (always do cleanups)
7011     local $SIG{INT} =
7012         local $SIG{TERM} =
7013         local $SIG{QUIT} =
7014         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7015
7016     # Note: $oldconf is undef if VM does not exists
7017     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7018     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7019     my $new_conf_raw = '';
7020
7021     my $rpcenv = PVE::RPCEnvironment::get();
7022     my $devinfo = {}; # info about drives included in backup
7023     my $virtdev_hash = {}; # info about allocated drives
7024
7025     eval {
7026         # enable interrupts
7027         local $SIG{INT} =
7028             local $SIG{TERM} =
7029             local $SIG{QUIT} =
7030             local $SIG{HUP} =
7031             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7032
7033         my $cfgfn = "$tmpdir/qemu-server.conf";
7034         my $firewall_config_fn = "$tmpdir/fw.conf";
7035         my $index_fn = "$tmpdir/index.json";
7036
7037         my $cmd = "restore";
7038
7039         my $param = [$pbs_backup_name, "index.json", $index_fn];
7040         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7041         my $index = PVE::Tools::file_get_contents($index_fn);
7042         $index = decode_json($index);
7043
7044         foreach my $info (@{$index->{files}}) {
7045             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7046                 my $devname = $1;
7047                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7048                     $devinfo->{$devname}->{size} = $1;
7049                 } else {
7050                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7051                 }
7052             }
7053         }
7054
7055         my $is_qemu_server_backup = scalar(
7056             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7057         );
7058         if (!$is_qemu_server_backup) {
7059             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7060         }
7061         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7062
7063         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7064         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7065
7066         if ($has_firewall_config) {
7067             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7068             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7069
7070             my $pve_firewall_dir = '/etc/pve/firewall';
7071             mkdir $pve_firewall_dir; # make sure the dir exists
7072             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7073         }
7074
7075         my $fh = IO::File->new($cfgfn, "r") ||
7076             die "unable to read qemu-server.conf - $!\n";
7077
7078         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7079
7080         # fixme: rate limit?
7081
7082         # create empty/temp config
7083         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7084
7085         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7086
7087         # allocate volumes
7088         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7089
7090         foreach my $virtdev (sort keys %$virtdev_hash) {
7091             my $d = $virtdev_hash->{$virtdev};
7092             next if $d->{is_cloudinit}; # no need to restore cloudinit
7093
7094             # this fails if storage is unavailable
7095             my $volid = $d->{volid};
7096             my $path = PVE::Storage::path($storecfg, $volid);
7097
7098             # for live-restore we only want to preload the efidisk and TPM state
7099             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7100
7101             my @ns_arg;
7102             if (defined(my $ns = $scfg->{namespace})) {
7103                 @ns_arg = ('--ns', $ns);
7104             }
7105
7106             my $pbs_restore_cmd = [
7107                 '/usr/bin/pbs-restore',
7108                 '--repository', $repo,
7109                 @ns_arg,
7110                 $pbs_backup_name,
7111                 "$d->{devname}.img.fidx",
7112                 $path,
7113                 '--verbose',
7114                 ];
7115
7116             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7117             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7118
7119             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7120                 push @$pbs_restore_cmd, '--skip-zero';
7121             }
7122
7123             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7124             print "restore proxmox backup image: $dbg_cmdstring\n";
7125             run_command($pbs_restore_cmd);
7126         }
7127
7128         $fh->seek(0, 0) || die "seek failed - $!\n";
7129
7130         my $cookie = { netcount => 0 };
7131         while (defined(my $line = <$fh>)) {
7132             $new_conf_raw .= restore_update_config_line(
7133                 $cookie,
7134                 $map,
7135                 $line,
7136                 $options->{unique},
7137             );
7138         }
7139
7140         $fh->close();
7141     };
7142     my $err = $@;
7143
7144     if ($err || !$options->{live}) {
7145         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7146     }
7147
7148     rmtree $tmpdir;
7149
7150     if ($err) {
7151         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7152         die $err;
7153     }
7154
7155     if ($options->{live}) {
7156         # keep lock during live-restore
7157         $new_conf_raw .= "\nlock: create";
7158     }
7159
7160     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7161     check_restore_permissions($rpcenv, $user, $new_conf);
7162     PVE::QemuConfig->write_config($vmid, $new_conf);
7163
7164     eval { rescan($vmid, 1); };
7165     warn $@ if $@;
7166
7167     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7168
7169     if ($options->{live}) {
7170         # enable interrupts
7171         local $SIG{INT} =
7172             local $SIG{TERM} =
7173             local $SIG{QUIT} =
7174             local $SIG{HUP} =
7175             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7176
7177         my $conf = PVE::QemuConfig->load_config($vmid);
7178         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7179
7180         # these special drives are already restored before start
7181         delete $devinfo->{'drive-efidisk0'};
7182         delete $devinfo->{'drive-tpmstate0-backup'};
7183
7184         my $pbs_opts = {
7185             repo => $repo,
7186             keyfile => $keyfile,
7187             snapshot => $pbs_backup_name,
7188             namespace => $namespace,
7189         };
7190         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7191
7192         PVE::QemuConfig->remove_lock($vmid, "create");
7193     }
7194 }
7195
7196 sub pbs_live_restore {
7197     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7198
7199     print "starting VM for live-restore\n";
7200     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7201
7202     my $pbs_backing = {};
7203     for my $ds (keys %$restored_disks) {
7204         $ds =~ m/^drive-(.*)$/;
7205         my $confname = $1;
7206         $pbs_backing->{$confname} = {
7207             repository => $opts->{repo},
7208             snapshot => $opts->{snapshot},
7209             archive => "$ds.img.fidx",
7210         };
7211         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7212         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7213
7214         my $drive = parse_drive($confname, $conf->{$confname});
7215         print "restoring '$ds' to '$drive->{file}'\n";
7216     }
7217
7218     my $drives_streamed = 0;
7219     eval {
7220         # make sure HA doesn't interrupt our restore by stopping the VM
7221         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7222             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7223         }
7224
7225         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7226         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7227         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7228
7229         my $qmeventd_fd = register_qmeventd_handle($vmid);
7230
7231         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7232         # this will effectively collapse the backing image chain consisting of
7233         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7234         # removes itself once all backing images vanish with 'auto-remove=on')
7235         my $jobs = {};
7236         for my $ds (sort keys %$restored_disks) {
7237             my $job_id = "restore-$ds";
7238             mon_cmd($vmid, 'block-stream',
7239                 'job-id' => $job_id,
7240                 device => "$ds",
7241             );
7242             $jobs->{$job_id} = {};
7243         }
7244
7245         mon_cmd($vmid, 'cont');
7246         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7247
7248         print "restore-drive jobs finished successfully, removing all tracking block devices"
7249             ." to disconnect from Proxmox Backup Server\n";
7250
7251         for my $ds (sort keys %$restored_disks) {
7252             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7253         }
7254
7255         close($qmeventd_fd);
7256     };
7257
7258     my $err = $@;
7259
7260     if ($err) {
7261         warn "An error occurred during live-restore: $err\n";
7262         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7263         die "live-restore failed\n";
7264     }
7265 }
7266
7267 sub restore_vma_archive {
7268     my ($archive, $vmid, $user, $opts, $comp) = @_;
7269
7270     my $readfrom = $archive;
7271
7272     my $cfg = PVE::Storage::config();
7273     my $commands = [];
7274     my $bwlimit = $opts->{bwlimit};
7275
7276     my $dbg_cmdstring = '';
7277     my $add_pipe = sub {
7278         my ($cmd) = @_;
7279         push @$commands, $cmd;
7280         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7281         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7282         $readfrom = '-';
7283     };
7284
7285     my $input = undef;
7286     if ($archive eq '-') {
7287         $input = '<&STDIN';
7288     } else {
7289         # If we use a backup from a PVE defined storage we also consider that
7290         # storage's rate limit:
7291         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7292         if (defined($volid)) {
7293             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7294             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7295             if ($readlimit) {
7296                 print STDERR "applying read rate limit: $readlimit\n";
7297                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7298                 $add_pipe->($cstream);
7299             }
7300         }
7301     }
7302
7303     if ($comp) {
7304         my $info = PVE::Storage::decompressor_info('vma', $comp);
7305         my $cmd = $info->{decompressor};
7306         push @$cmd, $readfrom;
7307         $add_pipe->($cmd);
7308     }
7309
7310     my $tmpdir = "/var/tmp/vzdumptmp$$";
7311     rmtree $tmpdir;
7312
7313     # disable interrupts (always do cleanups)
7314     local $SIG{INT} =
7315         local $SIG{TERM} =
7316         local $SIG{QUIT} =
7317         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7318
7319     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7320     POSIX::mkfifo($mapfifo, 0600);
7321     my $fifofh;
7322     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7323
7324     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7325
7326     my $devinfo = {}; # info about drives included in backup
7327     my $virtdev_hash = {}; # info about allocated drives
7328
7329     my $rpcenv = PVE::RPCEnvironment::get();
7330
7331     my $conffile = PVE::QemuConfig->config_file($vmid);
7332
7333     # Note: $oldconf is undef if VM does not exist
7334     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7335     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7336     my $new_conf_raw = '';
7337
7338     my %storage_limits;
7339
7340     my $print_devmap = sub {
7341         my $cfgfn = "$tmpdir/qemu-server.conf";
7342
7343         # we can read the config - that is already extracted
7344         my $fh = IO::File->new($cfgfn, "r") ||
7345             die "unable to read qemu-server.conf - $!\n";
7346
7347         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7348         if (-f $fwcfgfn) {
7349             my $pve_firewall_dir = '/etc/pve/firewall';
7350             mkdir $pve_firewall_dir; # make sure the dir exists
7351             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7352         }
7353
7354         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7355
7356         foreach my $info (values %{$virtdev_hash}) {
7357             my $storeid = $info->{storeid};
7358             next if defined($storage_limits{$storeid});
7359
7360             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7361             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7362             $storage_limits{$storeid} = $limit * 1024;
7363         }
7364
7365         foreach my $devname (keys %$devinfo) {
7366             die "found no device mapping information for device '$devname'\n"
7367                 if !$devinfo->{$devname}->{virtdev};
7368         }
7369
7370         # create empty/temp config
7371         if ($oldconf) {
7372             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7373             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7374         }
7375
7376         # allocate volumes
7377         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7378
7379         # print restore information to $fifofh
7380         foreach my $virtdev (sort keys %$virtdev_hash) {
7381             my $d = $virtdev_hash->{$virtdev};
7382             next if $d->{is_cloudinit}; # no need to restore cloudinit
7383
7384             my $storeid = $d->{storeid};
7385             my $volid = $d->{volid};
7386
7387             my $map_opts = '';
7388             if (my $limit = $storage_limits{$storeid}) {
7389                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7390             }
7391
7392             my $write_zeros = 1;
7393             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7394                 $write_zeros = 0;
7395             }
7396
7397             my $path = PVE::Storage::path($cfg, $volid);
7398
7399             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7400
7401             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7402         }
7403
7404         $fh->seek(0, 0) || die "seek failed - $!\n";
7405
7406         my $cookie = { netcount => 0 };
7407         while (defined(my $line = <$fh>)) {
7408             $new_conf_raw .= restore_update_config_line(
7409                 $cookie,
7410                 $map,
7411                 $line,
7412                 $opts->{unique},
7413             );
7414         }
7415
7416         $fh->close();
7417     };
7418
7419     my $oldtimeout;
7420
7421     eval {
7422         # enable interrupts
7423         local $SIG{INT} =
7424             local $SIG{TERM} =
7425             local $SIG{QUIT} =
7426             local $SIG{HUP} =
7427             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7428         local $SIG{ALRM} = sub { die "got timeout\n"; };
7429
7430         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7431
7432         my $parser = sub {
7433             my $line = shift;
7434
7435             print "$line\n";
7436
7437             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7438                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7439                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7440             } elsif ($line =~ m/^CTIME: /) {
7441                 # we correctly received the vma config, so we can disable
7442                 # the timeout now for disk allocation
7443                 alarm($oldtimeout || 0);
7444                 $oldtimeout = undef;
7445                 &$print_devmap();
7446                 print $fifofh "done\n";
7447                 close($fifofh);
7448                 $fifofh = undef;
7449             }
7450         };
7451
7452         print "restore vma archive: $dbg_cmdstring\n";
7453         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7454     };
7455     my $err = $@;
7456
7457     alarm($oldtimeout) if $oldtimeout;
7458
7459     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7460
7461     close($fifofh) if $fifofh;
7462     unlink $mapfifo;
7463     rmtree $tmpdir;
7464
7465     if ($err) {
7466         $restore_destroy_volumes->($cfg, $virtdev_hash);
7467         die $err;
7468     }
7469
7470     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7471     check_restore_permissions($rpcenv, $user, $new_conf);
7472     PVE::QemuConfig->write_config($vmid, $new_conf);
7473
7474     eval { rescan($vmid, 1); };
7475     warn $@ if $@;
7476
7477     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7478 }
7479
7480 sub restore_tar_archive {
7481     my ($archive, $vmid, $user, $opts) = @_;
7482
7483     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7484         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7485         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7486     }
7487
7488     if ($archive ne '-') {
7489         my $firstfile = tar_archive_read_firstfile($archive);
7490         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7491             if $firstfile ne 'qemu-server.conf';
7492     }
7493
7494     my $storecfg = PVE::Storage::config();
7495
7496     # avoid zombie disks when restoring over an existing VM -> cleanup first
7497     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7498     # skiplock=1 because qmrestore has set the 'create' lock itself already
7499     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7500     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7501
7502     my $tocmd = "/usr/lib/qemu-server/qmextract";
7503
7504     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7505     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7506     $tocmd .= ' --prealloc' if $opts->{prealloc};
7507     $tocmd .= ' --info' if $opts->{info};
7508
7509     # tar option "xf" does not autodetect compression when read from STDIN,
7510     # so we pipe to zcat
7511     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7512         PVE::Tools::shellquote("--to-command=$tocmd");
7513
7514     my $tmpdir = "/var/tmp/vzdumptmp$$";
7515     mkpath $tmpdir;
7516
7517     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7518     local $ENV{VZDUMP_VMID} = $vmid;
7519     local $ENV{VZDUMP_USER} = $user;
7520
7521     my $conffile = PVE::QemuConfig->config_file($vmid);
7522     my $new_conf_raw = '';
7523
7524     # disable interrupts (always do cleanups)
7525     local $SIG{INT} =
7526         local $SIG{TERM} =
7527         local $SIG{QUIT} =
7528         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7529
7530     eval {
7531         # enable interrupts
7532         local $SIG{INT} =
7533             local $SIG{TERM} =
7534             local $SIG{QUIT} =
7535             local $SIG{HUP} =
7536             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7537
7538         if ($archive eq '-') {
7539             print "extracting archive from STDIN\n";
7540             run_command($cmd, input => "<&STDIN");
7541         } else {
7542             print "extracting archive '$archive'\n";
7543             run_command($cmd);
7544         }
7545
7546         return if $opts->{info};
7547
7548         # read new mapping
7549         my $map = {};
7550         my $statfile = "$tmpdir/qmrestore.stat";
7551         if (my $fd = IO::File->new($statfile, "r")) {
7552             while (defined (my $line = <$fd>)) {
7553                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7554                     $map->{$1} = $2 if $1;
7555                 } else {
7556                     print STDERR "unable to parse line in statfile - $line\n";
7557                 }
7558             }
7559             $fd->close();
7560         }
7561
7562         my $confsrc = "$tmpdir/qemu-server.conf";
7563
7564         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7565
7566         my $cookie = { netcount => 0 };
7567         while (defined (my $line = <$srcfd>)) {
7568             $new_conf_raw .= restore_update_config_line(
7569                 $cookie,
7570                 $map,
7571                 $line,
7572                 $opts->{unique},
7573             );
7574         }
7575
7576         $srcfd->close();
7577     };
7578     if (my $err = $@) {
7579         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7580         die $err;
7581     }
7582
7583     rmtree $tmpdir;
7584
7585     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7586
7587     PVE::Cluster::cfs_update(); # make sure we read new file
7588
7589     eval { rescan($vmid, 1); };
7590     warn $@ if $@;
7591 };
7592
7593 sub foreach_storage_used_by_vm {
7594     my ($conf, $func) = @_;
7595
7596     my $sidhash = {};
7597
7598     PVE::QemuConfig->foreach_volume($conf, sub {
7599         my ($ds, $drive) = @_;
7600         return if drive_is_cdrom($drive);
7601
7602         my $volid = $drive->{file};
7603
7604         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7605         $sidhash->{$sid} = $sid if $sid;
7606     });
7607
7608     foreach my $sid (sort keys %$sidhash) {
7609         &$func($sid);
7610     }
7611 }
7612
7613 my $qemu_snap_storage = {
7614     rbd => 1,
7615 };
7616 sub do_snapshots_with_qemu {
7617     my ($storecfg, $volid, $deviceid) = @_;
7618
7619     return if $deviceid =~ m/tpmstate0/;
7620
7621     my $storage_name = PVE::Storage::parse_volume_id($volid);
7622     my $scfg = $storecfg->{ids}->{$storage_name};
7623     die "could not find storage '$storage_name'\n" if !defined($scfg);
7624
7625     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7626         return 1;
7627     }
7628
7629     if ($volid =~ m/\.(qcow2|qed)$/){
7630         return 1;
7631     }
7632
7633     return;
7634 }
7635
7636 sub qga_check_running {
7637     my ($vmid, $nowarn) = @_;
7638
7639     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7640     if ($@) {
7641         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7642         return 0;
7643     }
7644     return 1;
7645 }
7646
7647 sub template_create {
7648     my ($vmid, $conf, $disk) = @_;
7649
7650     my $storecfg = PVE::Storage::config();
7651
7652     PVE::QemuConfig->foreach_volume($conf, sub {
7653         my ($ds, $drive) = @_;
7654
7655         return if drive_is_cdrom($drive);
7656         return if $disk && $ds ne $disk;
7657
7658         my $volid = $drive->{file};
7659         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7660
7661         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7662         $drive->{file} = $voliddst;
7663         $conf->{$ds} = print_drive($drive);
7664         PVE::QemuConfig->write_config($vmid, $conf);
7665     });
7666 }
7667
7668 sub convert_iscsi_path {
7669     my ($path) = @_;
7670
7671     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7672         my $portal = $1;
7673         my $target = $2;
7674         my $lun = $3;
7675
7676         my $initiator_name = get_initiator_name();
7677
7678         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7679                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7680     }
7681
7682     die "cannot convert iscsi path '$path', unkown format\n";
7683 }
7684
7685 sub qemu_img_convert {
7686     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7687
7688     my $storecfg = PVE::Storage::config();
7689     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7690     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7691
7692     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7693
7694     my $cachemode;
7695     my $src_path;
7696     my $src_is_iscsi = 0;
7697     my $src_format;
7698
7699     if ($src_storeid) {
7700         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7701         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7702         $src_format = qemu_img_format($src_scfg, $src_volname);
7703         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7704         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7705         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7706     } elsif (-f $src_volid || -b $src_volid) {
7707         $src_path = $src_volid;
7708         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7709             $src_format = $1;
7710         }
7711     }
7712
7713     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7714
7715     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7716     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7717     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7718     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7719
7720     my $cmd = [];
7721     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7722     push @$cmd, '-l', "snapshot.name=$snapname"
7723         if $snapname && $src_format && $src_format eq "qcow2";
7724     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7725     push @$cmd, '-T', $cachemode if defined($cachemode);
7726     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7727
7728     if ($src_is_iscsi) {
7729         push @$cmd, '--image-opts';
7730         $src_path = convert_iscsi_path($src_path);
7731     } elsif ($src_format) {
7732         push @$cmd, '-f', $src_format;
7733     }
7734
7735     if ($dst_is_iscsi) {
7736         push @$cmd, '--target-image-opts';
7737         $dst_path = convert_iscsi_path($dst_path);
7738     } else {
7739         push @$cmd, '-O', $dst_format;
7740     }
7741
7742     push @$cmd, $src_path;
7743
7744     if (!$dst_is_iscsi && $is_zero_initialized) {
7745         push @$cmd, "zeroinit:$dst_path";
7746     } else {
7747         push @$cmd, $dst_path;
7748     }
7749
7750     my $parser = sub {
7751         my $line = shift;
7752         if($line =~ m/\((\S+)\/100\%\)/){
7753             my $percent = $1;
7754             my $transferred = int($size * $percent / 100);
7755             my $total_h = render_bytes($size, 1);
7756             my $transferred_h = render_bytes($transferred, 1);
7757
7758             print "transferred $transferred_h of $total_h ($percent%)\n";
7759         }
7760
7761     };
7762
7763     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7764     my $err = $@;
7765     die "copy failed: $err" if $err;
7766 }
7767
7768 sub qemu_img_format {
7769     my ($scfg, $volname) = @_;
7770
7771     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7772         return $1;
7773     } else {
7774         return "raw";
7775     }
7776 }
7777
7778 sub qemu_drive_mirror {
7779     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7780
7781     $jobs = {} if !$jobs;
7782
7783     my $qemu_target;
7784     my $format;
7785     $jobs->{"drive-$drive"} = {};
7786
7787     if ($dst_volid =~ /^nbd:/) {
7788         $qemu_target = $dst_volid;
7789         $format = "nbd";
7790     } else {
7791         my $storecfg = PVE::Storage::config();
7792         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7793
7794         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7795
7796         $format = qemu_img_format($dst_scfg, $dst_volname);
7797
7798         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7799
7800         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7801     }
7802
7803     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7804     $opts->{format} = $format if $format;
7805
7806     if (defined($src_bitmap)) {
7807         $opts->{sync} = 'incremental';
7808         $opts->{bitmap} = $src_bitmap;
7809         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7810     }
7811
7812     if (defined($bwlimit)) {
7813         $opts->{speed} = $bwlimit * 1024;
7814         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7815     } else {
7816         print "drive mirror is starting for drive-$drive\n";
7817     }
7818
7819     # if a job already runs for this device we get an error, catch it for cleanup
7820     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7821     if (my $err = $@) {
7822         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7823         warn "$@\n" if $@;
7824         die "mirroring error: $err\n";
7825     }
7826
7827     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7828 }
7829
7830 # $completion can be either
7831 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7832 # 'cancel': wait until all jobs are ready, block-job-cancel them
7833 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7834 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7835 sub qemu_drive_mirror_monitor {
7836     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7837
7838     $completion //= 'complete';
7839     $op //= "mirror";
7840
7841     eval {
7842         my $err_complete = 0;
7843
7844         my $starttime = time ();
7845         while (1) {
7846             die "block job ('$op') timed out\n" if $err_complete > 300;
7847
7848             my $stats = mon_cmd($vmid, "query-block-jobs");
7849             my $ctime = time();
7850
7851             my $running_jobs = {};
7852             for my $stat (@$stats) {
7853                 next if $stat->{type} ne $op;
7854                 $running_jobs->{$stat->{device}} = $stat;
7855             }
7856
7857             my $readycounter = 0;
7858
7859             for my $job_id (sort keys %$jobs) {
7860                 my $job = $running_jobs->{$job_id};
7861
7862                 my $vanished = !defined($job);
7863                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7864                 if($complete || ($vanished && $completion eq 'auto')) {
7865                     print "$job_id: $op-job finished\n";
7866                     delete $jobs->{$job_id};
7867                     next;
7868                 }
7869
7870                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7871
7872                 my $busy = $job->{busy};
7873                 my $ready = $job->{ready};
7874                 if (my $total = $job->{len}) {
7875                     my $transferred = $job->{offset} || 0;
7876                     my $remaining = $total - $transferred;
7877                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7878
7879                     my $duration = $ctime - $starttime;
7880                     my $total_h = render_bytes($total, 1);
7881                     my $transferred_h = render_bytes($transferred, 1);
7882
7883                     my $status = sprintf(
7884                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7885                         render_duration($duration),
7886                     );
7887
7888                     if ($ready) {
7889                         if ($busy) {
7890                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7891                         } else {
7892                             $status .= ", ready";
7893                         }
7894                     }
7895                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7896                     $jobs->{$job_id}->{ready} = $ready;
7897                 }
7898
7899                 $readycounter++ if $job->{ready};
7900             }
7901
7902             last if scalar(keys %$jobs) == 0;
7903
7904             if ($readycounter == scalar(keys %$jobs)) {
7905                 print "all '$op' jobs are ready\n";
7906
7907                 # do the complete later (or has already been done)
7908                 last if $completion eq 'skip' || $completion eq 'auto';
7909
7910                 if ($vmiddst && $vmiddst != $vmid) {
7911                     my $agent_running = $qga && qga_check_running($vmid);
7912                     if ($agent_running) {
7913                         print "freeze filesystem\n";
7914                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7915                         warn $@ if $@;
7916                     } else {
7917                         print "suspend vm\n";
7918                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7919                         warn $@ if $@;
7920                     }
7921
7922                     # if we clone a disk for a new target vm, we don't switch the disk
7923                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7924
7925                     if ($agent_running) {
7926                         print "unfreeze filesystem\n";
7927                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7928                         warn $@ if $@;
7929                     } else {
7930                         print "resume vm\n";
7931                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7932                         warn $@ if $@;
7933                     }
7934
7935                     last;
7936                 } else {
7937
7938                     for my $job_id (sort keys %$jobs) {
7939                         # try to switch the disk if source and destination are on the same guest
7940                         print "$job_id: Completing block job_id...\n";
7941
7942                         my $op;
7943                         if ($completion eq 'complete') {
7944                             $op = 'block-job-complete';
7945                         } elsif ($completion eq 'cancel') {
7946                             $op = 'block-job-cancel';
7947                         } else {
7948                             die "invalid completion value: $completion\n";
7949                         }
7950                         eval { mon_cmd($vmid, $op, device => $job_id) };
7951                         if ($@ =~ m/cannot be completed/) {
7952                             print "$job_id: block job cannot be completed, trying again.\n";
7953                             $err_complete++;
7954                         }else {
7955                             print "$job_id: Completed successfully.\n";
7956                             $jobs->{$job_id}->{complete} = 1;
7957                         }
7958                     }
7959                 }
7960             }
7961             sleep 1;
7962         }
7963     };
7964     my $err = $@;
7965
7966     if ($err) {
7967         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7968         die "block job ($op) error: $err";
7969     }
7970 }
7971
7972 sub qemu_blockjobs_cancel {
7973     my ($vmid, $jobs) = @_;
7974
7975     foreach my $job (keys %$jobs) {
7976         print "$job: Cancelling block job\n";
7977         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7978         $jobs->{$job}->{cancel} = 1;
7979     }
7980
7981     while (1) {
7982         my $stats = mon_cmd($vmid, "query-block-jobs");
7983
7984         my $running_jobs = {};
7985         foreach my $stat (@$stats) {
7986             $running_jobs->{$stat->{device}} = $stat;
7987         }
7988
7989         foreach my $job (keys %$jobs) {
7990
7991             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7992                 print "$job: Done.\n";
7993                 delete $jobs->{$job};
7994             }
7995         }
7996
7997         last if scalar(keys %$jobs) == 0;
7998
7999         sleep 1;
8000     }
8001 }
8002
8003 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8004 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8005 my sub clone_disk_check_io_uring {
8006     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8007
8008     return if !$use_drive_mirror;
8009
8010     # Don't complain when not changing storage.
8011     # Assume if it works for the source, it'll work for the target too.
8012     return if $src_storeid eq $dst_storeid;
8013
8014     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8015     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8016
8017     my $cache_direct = drive_uses_cache_direct($src_drive);
8018
8019     my $src_uses_io_uring;
8020     if ($src_drive->{aio}) {
8021         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8022     } else {
8023         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8024     }
8025
8026     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8027         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8028 }
8029
8030 sub clone_disk {
8031     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8032
8033     my ($vmid, $running) = $source->@{qw(vmid running)};
8034     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8035
8036     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8037     my ($storage, $format) = $dest->@{qw(storage format)};
8038
8039     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8040
8041     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8042         die "cloning from/to EFI disk requires EFI disk\n"
8043             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8044         die "cloning from/to TPM state requires TPM state\n"
8045             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8046
8047         # This would lead to two device nodes in QEMU pointing to the same backing image!
8048         die "cannot change drive name when cloning disk from/to the same VM\n"
8049             if $use_drive_mirror && $vmid == $newvmid;
8050     }
8051
8052     die "cannot move TPM state while VM is running\n"
8053         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8054
8055     my $newvolid;
8056
8057     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8058     print "$src_drivename " if $src_drivename;
8059     print "($drive->{file})\n";
8060
8061     if (!$full) {
8062         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8063         push @$newvollist, $newvolid;
8064     } else {
8065         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8066         my $storeid = $storage || $src_storeid;
8067
8068         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8069
8070         my $name = undef;
8071         my $size = undef;
8072         if (drive_is_cloudinit($drive)) {
8073             $name = "vm-$newvmid-cloudinit";
8074             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8075             if ($scfg->{path}) {
8076                 $name .= ".$dst_format";
8077             }
8078             $snapname = undef;
8079             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8080         } elsif ($dst_drivename eq 'efidisk0') {
8081             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8082         } elsif ($dst_drivename eq 'tpmstate0') {
8083             $dst_format = 'raw';
8084             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8085         } else {
8086             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8087
8088             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8089         }
8090         $newvolid = PVE::Storage::vdisk_alloc(
8091             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8092         );
8093         push @$newvollist, $newvolid;
8094
8095         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8096
8097         if (drive_is_cloudinit($drive)) {
8098             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8099             # if this is the case, we have to complete any block-jobs still there from
8100             # previous drive-mirrors
8101             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8102                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8103             }
8104             goto no_data_clone;
8105         }
8106
8107         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8108         if ($use_drive_mirror) {
8109             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8110                 $completion, $qga, $bwlimit);
8111         } else {
8112             if ($dst_drivename eq 'efidisk0') {
8113                 # the relevant data on the efidisk may be smaller than the source
8114                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8115                 # that is given by the OVMF_VARS.fd
8116                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8117                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8118
8119                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8120
8121                 # better for Ceph if block size is not too small, see bug #3324
8122                 my $bs = 1024*1024;
8123
8124                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8125
8126                 if ($src_format eq 'qcow2' && $snapname) {
8127                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8128                         if !min_version(kvm_user_version(), 6, 2);
8129                     push $cmd->@*, '-l', $snapname;
8130                 }
8131                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8132                 run_command($cmd);
8133             } else {
8134                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8135             }
8136         }
8137     }
8138
8139 no_data_clone:
8140     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8141
8142     my $disk = dclone($drive);
8143     delete $disk->{format};
8144     $disk->{file} = $newvolid;
8145     $disk->{size} = $size if defined($size);
8146
8147     return $disk;
8148 }
8149
8150 sub get_running_qemu_version {
8151     my ($vmid) = @_;
8152     my $res = mon_cmd($vmid, "query-version");
8153     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8154 }
8155
8156 sub qemu_use_old_bios_files {
8157     my ($machine_type) = @_;
8158
8159     return if !$machine_type;
8160
8161     my $use_old_bios_files = undef;
8162
8163     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8164         $machine_type = $1;
8165         $use_old_bios_files = 1;
8166     } else {
8167         my $version = extract_version($machine_type, kvm_user_version());
8168         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8169         # load new efi bios files on migration. So this hack is required to allow
8170         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8171         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8172         $use_old_bios_files = !min_version($version, 2, 4);
8173     }
8174
8175     return ($use_old_bios_files, $machine_type);
8176 }
8177
8178 sub get_efivars_size {
8179     my ($conf, $efidisk) = @_;
8180
8181     my $arch = get_vm_arch($conf);
8182     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8183     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8184     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8185     return -s $ovmf_vars;
8186 }
8187
8188 sub update_efidisk_size {
8189     my ($conf) = @_;
8190
8191     return if !defined($conf->{efidisk0});
8192
8193     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8194     $disk->{size} = get_efivars_size($conf);
8195     $conf->{efidisk0} = print_drive($disk);
8196
8197     return;
8198 }
8199
8200 sub update_tpmstate_size {
8201     my ($conf) = @_;
8202
8203     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8204     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8205     $conf->{tpmstate0} = print_drive($disk);
8206 }
8207
8208 sub create_efidisk($$$$$$$) {
8209     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8210
8211     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8212
8213     my $vars_size_b = -s $ovmf_vars;
8214     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8215     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8216     PVE::Storage::activate_volumes($storecfg, [$volid]);
8217
8218     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8219     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8220
8221     return ($volid, $size/1024);
8222 }
8223
8224 sub vm_iothreads_list {
8225     my ($vmid) = @_;
8226
8227     my $res = mon_cmd($vmid, 'query-iothreads');
8228
8229     my $iothreads = {};
8230     foreach my $iothread (@$res) {
8231         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8232     }
8233
8234     return $iothreads;
8235 }
8236
8237 sub scsihw_infos {
8238     my ($conf, $drive) = @_;
8239
8240     my $maxdev = 0;
8241
8242     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8243         $maxdev = 7;
8244     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8245         $maxdev = 1;
8246     } else {
8247         $maxdev = 256;
8248     }
8249
8250     my $controller = int($drive->{index} / $maxdev);
8251     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8252         ? "virtioscsi"
8253         : "scsihw";
8254
8255     return ($maxdev, $controller, $controller_prefix);
8256 }
8257
8258 sub resolve_dst_disk_format {
8259         my ($storecfg, $storeid, $src_volname, $format) = @_;
8260         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8261
8262         if (!$format) {
8263             # if no target format is specified, use the source disk format as hint
8264             if ($src_volname) {
8265                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8266                 $format = qemu_img_format($scfg, $src_volname);
8267             } else {
8268                 return $defFormat;
8269             }
8270         }
8271
8272         # test if requested format is supported - else use default
8273         my $supported = grep { $_ eq $format } @$validFormats;
8274         $format = $defFormat if !$supported;
8275         return $format;
8276 }
8277
8278 # NOTE: if this logic changes, please update docs & possibly gui logic
8279 sub find_vmstate_storage {
8280     my ($conf, $storecfg) = @_;
8281
8282     # first, return storage from conf if set
8283     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8284
8285     my ($target, $shared, $local);
8286
8287     foreach_storage_used_by_vm($conf, sub {
8288         my ($sid) = @_;
8289         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8290         my $dst = $scfg->{shared} ? \$shared : \$local;
8291         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8292     });
8293
8294     # second, use shared storage where VM has at least one disk
8295     # third, use local storage where VM has at least one disk
8296     # fall back to local storage
8297     $target = $shared // $local // 'local';
8298
8299     return $target;
8300 }
8301
8302 sub generate_uuid {
8303     my ($uuid, $uuid_str);
8304     UUID::generate($uuid);
8305     UUID::unparse($uuid, $uuid_str);
8306     return $uuid_str;
8307 }
8308
8309 sub generate_smbios1_uuid {
8310     return "uuid=".generate_uuid();
8311 }
8312
8313 sub nbd_stop {
8314     my ($vmid) = @_;
8315
8316     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8317 }
8318
8319 sub create_reboot_request {
8320     my ($vmid) = @_;
8321     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8322         or die "failed to create reboot trigger file: $!\n";
8323     close($fh);
8324 }
8325
8326 sub clear_reboot_request {
8327     my ($vmid) = @_;
8328     my $path = "/run/qemu-server/$vmid.reboot";
8329     my $res = 0;
8330
8331     $res = unlink($path);
8332     die "could not remove reboot request for $vmid: $!"
8333         if !$res && $! != POSIX::ENOENT;
8334
8335     return $res;
8336 }
8337
8338 sub bootorder_from_legacy {
8339     my ($conf, $bootcfg) = @_;
8340
8341     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8342     my $bootindex_hash = {};
8343     my $i = 1;
8344     foreach my $o (split(//, $boot)) {
8345         $bootindex_hash->{$o} = $i*100;
8346         $i++;
8347     }
8348
8349     my $bootorder = {};
8350
8351     PVE::QemuConfig->foreach_volume($conf, sub {
8352         my ($ds, $drive) = @_;
8353
8354         if (drive_is_cdrom ($drive, 1)) {
8355             if ($bootindex_hash->{d}) {
8356                 $bootorder->{$ds} = $bootindex_hash->{d};
8357                 $bootindex_hash->{d} += 1;
8358             }
8359         } elsif ($bootindex_hash->{c}) {
8360             $bootorder->{$ds} = $bootindex_hash->{c}
8361                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8362             $bootindex_hash->{c} += 1;
8363         }
8364     });
8365
8366     if ($bootindex_hash->{n}) {
8367         for (my $i = 0; $i < $MAX_NETS; $i++) {
8368             my $netname = "net$i";
8369             next if !$conf->{$netname};
8370             $bootorder->{$netname} = $bootindex_hash->{n};
8371             $bootindex_hash->{n} += 1;
8372         }
8373     }
8374
8375     return $bootorder;
8376 }
8377
8378 # Generate default device list for 'boot: order=' property. Matches legacy
8379 # default boot order, but with explicit device names. This is important, since
8380 # the fallback for when neither 'order' nor the old format is specified relies
8381 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8382 sub get_default_bootdevices {
8383     my ($conf) = @_;
8384
8385     my @ret = ();
8386
8387     # harddisk
8388     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8389     push @ret, $first if $first;
8390
8391     # cdrom
8392     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8393     push @ret, $first if $first;
8394
8395     # network
8396     for (my $i = 0; $i < $MAX_NETS; $i++) {
8397         my $netname = "net$i";
8398         next if !$conf->{$netname};
8399         push @ret, $netname;
8400         last;
8401     }
8402
8403     return \@ret;
8404 }
8405
8406 sub device_bootorder {
8407     my ($conf) = @_;
8408
8409     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8410
8411     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8412
8413     my $bootorder = {};
8414     if (!defined($boot) || $boot->{legacy}) {
8415         $bootorder = bootorder_from_legacy($conf, $boot);
8416     } elsif ($boot->{order}) {
8417         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8418         for my $dev (PVE::Tools::split_list($boot->{order})) {
8419             $bootorder->{$dev} = $i++;
8420         }
8421     }
8422
8423     return $bootorder;
8424 }
8425
8426 sub register_qmeventd_handle {
8427     my ($vmid) = @_;
8428
8429     my $fh;
8430     my $peer = "/var/run/qmeventd.sock";
8431     my $count = 0;
8432
8433     for (;;) {
8434         $count++;
8435         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8436         last if $fh;
8437         if ($! != EINTR && $! != EAGAIN) {
8438             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8439         }
8440         if ($count > 4) {
8441             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8442               . "after $count retries\n";
8443         }
8444         usleep(25000);
8445     }
8446
8447     # send handshake to mark VM as backing up
8448     print $fh to_json({vzdump => {vmid => "$vmid"}});
8449
8450     # return handle to be closed later when inhibit is no longer required
8451     return $fh;
8452 }
8453
8454 # bash completion helper
8455
8456 sub complete_backup_archives {
8457     my ($cmdname, $pname, $cvalue) = @_;
8458
8459     my $cfg = PVE::Storage::config();
8460
8461     my $storeid;
8462
8463     if ($cvalue =~ m/^([^:]+):/) {
8464         $storeid = $1;
8465     }
8466
8467     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8468
8469     my $res = [];
8470     foreach my $id (keys %$data) {
8471         foreach my $item (@{$data->{$id}}) {
8472             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8473             push @$res, $item->{volid} if defined($item->{volid});
8474         }
8475     }
8476
8477     return $res;
8478 }
8479
8480 my $complete_vmid_full = sub {
8481     my ($running) = @_;
8482
8483     my $idlist = vmstatus();
8484
8485     my $res = [];
8486
8487     foreach my $id (keys %$idlist) {
8488         my $d = $idlist->{$id};
8489         if (defined($running)) {
8490             next if $d->{template};
8491             next if $running && $d->{status} ne 'running';
8492             next if !$running && $d->{status} eq 'running';
8493         }
8494         push @$res, $id;
8495
8496     }
8497     return $res;
8498 };
8499
8500 sub complete_vmid {
8501     return &$complete_vmid_full();
8502 }
8503
8504 sub complete_vmid_stopped {
8505     return &$complete_vmid_full(0);
8506 }
8507
8508 sub complete_vmid_running {
8509     return &$complete_vmid_full(1);
8510 }
8511
8512 sub complete_storage {
8513
8514     my $cfg = PVE::Storage::config();
8515     my $ids = $cfg->{ids};
8516
8517     my $res = [];
8518     foreach my $sid (keys %$ids) {
8519         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8520         next if !$ids->{$sid}->{content}->{images};
8521         push @$res, $sid;
8522     }
8523
8524     return $res;
8525 }
8526
8527 sub complete_migration_storage {
8528     my ($cmd, $param, $current_value, $all_args) = @_;
8529
8530     my $targetnode = @$all_args[1];
8531
8532     my $cfg = PVE::Storage::config();
8533     my $ids = $cfg->{ids};
8534
8535     my $res = [];
8536     foreach my $sid (keys %$ids) {
8537         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8538         next if !$ids->{$sid}->{content}->{images};
8539         push @$res, $sid;
8540     }
8541
8542     return $res;
8543 }
8544
8545 sub vm_is_paused {
8546     my ($vmid, $include_suspended) = @_;
8547     my $qmpstatus = eval {
8548         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8549         mon_cmd($vmid, "query-status");
8550     };
8551     warn "$@\n" if $@;
8552     return $qmpstatus && (
8553         $qmpstatus->{status} eq "paused" ||
8554         $qmpstatus->{status} eq "prelaunch" ||
8555         ($include_suspended && $qmpstatus->{status} eq "suspended")
8556     );
8557 }
8558
8559 sub check_volume_storage_type {
8560     my ($storecfg, $vol) = @_;
8561
8562     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8563     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8564     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8565
8566     die "storage '$storeid' does not support content-type '$vtype'\n"
8567         if !$scfg->{content}->{$vtype};
8568
8569     return 1;
8570 }
8571
8572 sub add_nets_bridge_fdb {
8573     my ($conf, $vmid) = @_;
8574
8575     for my $opt (keys %$conf) {
8576         next if $opt !~ m/^net(\d+)$/;
8577         my $iface = "tap${vmid}i$1";
8578         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8579         my $net = parse_net($conf->{$opt}, 1) or next;
8580
8581         my $mac = $net->{macaddr};
8582         if (!$mac) {
8583             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8584                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8585             next;
8586         }
8587
8588         my $bridge = $net->{bridge};
8589         if (!$bridge) {
8590             log_warn("Interface '$iface' not attached to any bridge.");
8591             next;
8592         }
8593         if ($have_sdn) {
8594             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8595         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8596             PVE::Network::add_bridge_fdb($iface, $mac);
8597         }
8598     }
8599 }
8600
8601 sub del_nets_bridge_fdb {
8602     my ($conf, $vmid) = @_;
8603
8604     for my $opt (keys %$conf) {
8605         next if $opt !~ m/^net(\d+)$/;
8606         my $iface = "tap${vmid}i$1";
8607
8608         my $net = parse_net($conf->{$opt}) or next;
8609         my $mac = $net->{macaddr} or next;
8610
8611         my $bridge = $net->{bridge};
8612         if ($have_sdn) {
8613             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8614         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8615             PVE::Network::del_bridge_fdb($iface, $mac);
8616         }
8617     }
8618 }
8619
8620 sub create_ifaces_ipams_ips {
8621     my ($conf, $vmid) = @_;
8622
8623     return if !$have_sdn;
8624
8625     foreach my $opt (keys %$conf) {
8626         if ($opt =~ m/^net(\d+)$/) {
8627             my $value = $conf->{$opt};
8628             my $net = PVE::QemuServer::parse_net($value);
8629             eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8630             warn $@ if $@;
8631         }
8632     }
8633 }
8634
8635 sub delete_ifaces_ipams_ips {
8636     my ($conf, $vmid) = @_;
8637
8638     return if !$have_sdn;
8639
8640     foreach my $opt (keys %$conf) {
8641         if ($opt =~ m/^net(\d+)$/) {
8642             my $net = PVE::QemuServer::parse_net($conf->{$opt});
8643             eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8644             warn $@ if $@;
8645         }
8646     }
8647 }
8648
8649 1;