PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory qw(get_current_memory);
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  62 use PVE::QemuServer::USB;
  63
  64 my $have_sdn;
  65 eval {
  66     require PVE::Network::SDN::Zones;
  67     require PVE::Network::SDN::Vnets;
  68     $have_sdn = 1;
  69 };
  70
  71 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  72 my $OVMF = {
  73     x86_64 => {
  74         '4m-no-smm' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  77         ],
  78         '4m-no-smm-ms' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  81         ],
  82         '4m' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  85         ],
  86         '4m-ms' => [
  87             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  89         ],
  90         # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
  91         # anymore. how can we deperacate this sanely without breaking existing instances, or using
  92         # older backups and snapshot?
  93         default => [
  94             "$EDK2_FW_BASE/OVMF_CODE.fd",
  95             "$EDK2_FW_BASE/OVMF_VARS.fd",
  96         ],
  97     },
  98     aarch64 => {
  99         default => [
 100             "$EDK2_FW_BASE/AAVMF_CODE.fd",
 101             "$EDK2_FW_BASE/AAVMF_VARS.fd",
 102         ],
 103     },
 104 };
 105
 106 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 107
 108 # Note about locking: we use flock on the config file protect against concurent actions.
 109 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 110 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 111 # But you can ignore this kind of lock with the --skiplock flag.
 112
 113 cfs_register_file(
 114     '/qemu-server/',
 115     \&parse_vm_config,
 116     \&write_vm_config
 117 );
 118
 119 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 120     description => "Some command save/restore state from this location.",
 121     type => 'string',
 122     maxLength => 128,
 123     optional => 1,
 124 });
 125
 126 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 127         description => "Specifies the QEMU machine type.",
 128         type => 'string',
 129         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 130         maxLength => 40,
 131         optional => 1,
 132 });
 133
 134 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 135 my $nodename_cache;
 136 sub nodename {
 137     $nodename_cache //= PVE::INotify::nodename();
 138     return $nodename_cache;
 139 }
 140
 141 my $watchdog_fmt = {
 142     model => {
 143         default_key => 1,
 144         type => 'string',
 145         enum => [qw(i6300esb ib700)],
 146         description => "Watchdog type to emulate.",
 147         default => 'i6300esb',
 148         optional => 1,
 149     },
 150     action => {
 151         type => 'string',
 152         enum => [qw(reset shutdown poweroff pause debug none)],
 153         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 154         optional => 1,
 155     },
 156 };
 157 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 158
 159 my $agent_fmt = {
 160     enabled => {
 161         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 162         type => 'boolean',
 163         default => 0,
 164         default_key => 1,
 165     },
 166     fstrim_cloned_disks => {
 167         description => "Run fstrim after moving a disk or migrating the VM.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0,
 171     },
 172     'freeze-fs-on-backup' => {
 173         description => "Freeze/thaw guest filesystems on backup for consistency.",
 174         type => 'boolean',
 175         optional => 1,
 176         default => 1,
 177     },
 178     type => {
 179         description => "Select the agent type",
 180         type => 'string',
 181         default => 'virtio',
 182         optional => 1,
 183         enum => [qw(virtio isa)],
 184     },
 185 };
 186
 187 my $vga_fmt = {
 188     type => {
 189         description => "Select the VGA type.",
 190         type => 'string',
 191         default => 'std',
 192         optional => 1,
 193         default_key => 1,
 194         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 195     },
 196     memory => {
 197         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 198         type => 'integer',
 199         optional => 1,
 200         minimum => 4,
 201         maximum => 512,
 202     },
 203     clipboard => {
 204         description => 'Enable a specific clipboard. If not set, depending on the display type the'
 205             .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
 206         type => 'string',
 207         enum => ['vnc'],
 208         optional => 1,
 209     },
 210 };
 211
 212 my $ivshmem_fmt = {
 213     size => {
 214         type => 'integer',
 215         minimum => 1,
 216         description => "The size of the file in MB.",
 217     },
 218     name => {
 219         type => 'string',
 220         pattern => '[a-zA-Z0-9\-]+',
 221         optional => 1,
 222         format_description => 'string',
 223         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 224     },
 225 };
 226
 227 my $audio_fmt = {
 228     device => {
 229         type => 'string',
 230         enum => [qw(ich9-intel-hda intel-hda AC97)],
 231         description =>  "Configure an audio device."
 232     },
 233     driver =>  {
 234         type => 'string',
 235         enum => ['spice', 'none'],
 236         default => 'spice',
 237         optional => 1,
 238         description => "Driver backend for the audio device."
 239     },
 240 };
 241
 242 my $spice_enhancements_fmt = {
 243     foldersharing => {
 244         type => 'boolean',
 245         optional => 1,
 246         default => '0',
 247         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 248     },
 249     videostreaming =>  {
 250         type => 'string',
 251         enum => ['off', 'all', 'filter'],
 252         default => 'off',
 253         optional => 1,
 254         description => "Enable video streaming. Uses compression for detected video streams."
 255     },
 256 };
 257
 258 my $rng_fmt = {
 259     source => {
 260         type => 'string',
 261         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 262         default_key => 1,
 263         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 264             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 265             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 266             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 267             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 268             ." a hardware RNG from the host.",
 269     },
 270     max_bytes => {
 271         type => 'integer',
 272         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 273             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 274             ." `0` to disable limiting (potentially dangerous!).",
 275         optional => 1,
 276
 277         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 278         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 279         # reading from /dev/urandom
 280         default => 1024,
 281     },
 282     period => {
 283         type => 'integer',
 284         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 285             ." the guest to retrieve another 'max_bytes' of entropy.",
 286         optional => 1,
 287         default => 1000,
 288     },
 289 };
 290
 291 my $meta_info_fmt = {
 292     'ctime' => {
 293         type => 'integer',
 294         description => "The guest creation timestamp as UNIX epoch time",
 295         minimum => 0,
 296         optional => 1,
 297     },
 298     'creation-qemu' => {
 299         type => 'string',
 300         description => "The QEMU (machine) version from the time this VM was created.",
 301         pattern => '\d+(\.\d+)+',
 302         optional => 1,
 303     },
 304 };
 305
 306 my $confdesc = {
 307     onboot => {
 308         optional => 1,
 309         type => 'boolean',
 310         description => "Specifies whether a VM will be started during system bootup.",
 311         default => 0,
 312     },
 313     autostart => {
 314         optional => 1,
 315         type => 'boolean',
 316         description => "Automatic restart after crash (currently ignored).",
 317         default => 0,
 318     },
 319     hotplug => {
 320         optional => 1,
 321         type => 'string', format => 'pve-hotplug-features',
 322         description => "Selectively enable hotplug features. This is a comma separated list of"
 323             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 324             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 325             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 326             ." windows > 7.",
 327         default => 'network,disk,usb',
 328     },
 329     reboot => {
 330         optional => 1,
 331         type => 'boolean',
 332         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 333         default => 1,
 334     },
 335     lock => {
 336         optional => 1,
 337         type => 'string',
 338         description => "Lock/unlock the VM.",
 339         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 340     },
 341     cpulimit => {
 342         optional => 1,
 343         type => 'number',
 344         description => "Limit of CPU usage.",
 345         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 346             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 347         minimum => 0,
 348         maximum => 128,
 349         default => 0,
 350     },
 351     cpuunits => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 355         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 356             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 357             ." weights of all the other running VMs.",
 358         minimum => 1,
 359         maximum => 262144,
 360         default => 'cgroup v1: 1024, cgroup v2: 100',
 361     },
 362     memory => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Memory properties.",
 366         format => $PVE::QemuServer::Memory::memory_fmt
 367     },
 368     balloon => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 372         minimum => 0,
 373     },
 374     shares => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 378             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 379             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 380         minimum => 0,
 381         maximum => 50000,
 382         default => 1000,
 383     },
 384     keyboard => {
 385         optional => 1,
 386         type => 'string',
 387         description => "Keyboard layout for VNC server. This option is generally not required and"
 388         ." is often better handled from within the guest OS.",
 389         enum => PVE::Tools::kvmkeymaplist(),
 390         default => undef,
 391     },
 392     name => {
 393         optional => 1,
 394         type => 'string', format => 'dns-name',
 395         description => "Set a name for the VM. Only used on the configuration web interface.",
 396     },
 397     scsihw => {
 398         optional => 1,
 399         type => 'string',
 400         description => "SCSI controller model",
 401         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 402         default => 'lsi',
 403     },
 404     description => {
 405         optional => 1,
 406         type => 'string',
 407         description => "Description for the VM. Shown in the web-interface VM's summary."
 408             ." This is saved as comment inside the configuration file.",
 409         maxLength => 1024 * 8,
 410     },
 411     ostype => {
 412         optional => 1,
 413         type => 'string',
 414         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 415         description => "Specify guest operating system.",
 416         verbose_description => <<EODESC,
 417 Specify guest operating system. This is used to enable special
 418 optimization/features for specific operating systems:
 419
 420 [horizontal]
 421 other;; unspecified OS
 422 wxp;; Microsoft Windows XP
 423 w2k;; Microsoft Windows 2000
 424 w2k3;; Microsoft Windows 2003
 425 w2k8;; Microsoft Windows 2008
 426 wvista;; Microsoft Windows Vista
 427 win7;; Microsoft Windows 7
 428 win8;; Microsoft Windows 8/2012/2012r2
 429 win10;; Microsoft Windows 10/2016/2019
 430 win11;; Microsoft Windows 11/2022
 431 l24;; Linux 2.4 Kernel
 432 l26;; Linux 2.6 - 6.X Kernel
 433 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 434 EODESC
 435     },
 436     boot => {
 437         optional => 1,
 438         type => 'string', format => 'pve-qm-boot',
 439         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 440             ." key or 'legacy=' is deprecated.",
 441     },
 442     bootdisk => {
 443         optional => 1,
 444         type => 'string', format => 'pve-qm-bootdisk',
 445         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 446         pattern => '(ide|sata|scsi|virtio)\d+',
 447     },
 448     smp => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "The number of CPUs. Please use option -sockets instead.",
 452         minimum => 1,
 453         default => 1,
 454     },
 455     sockets => {
 456         optional => 1,
 457         type => 'integer',
 458         description => "The number of CPU sockets.",
 459         minimum => 1,
 460         default => 1,
 461     },
 462     cores => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "The number of cores per socket.",
 466         minimum => 1,
 467         default => 1,
 468     },
 469     numa => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable NUMA.",
 473         default => 0,
 474     },
 475     hugepages => {
 476         optional => 1,
 477         type => 'string',
 478         description => "Enable/disable hugepages memory.",
 479         enum => [qw(any 2 1024)],
 480     },
 481     keephugepages => {
 482         optional => 1,
 483         type => 'boolean',
 484         default => 0,
 485         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 486             ." after VM shutdown and can be used for subsequent starts.",
 487     },
 488     vcpus => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Number of hotplugged vcpus.",
 492         minimum => 1,
 493         default => 0,
 494     },
 495     acpi => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable ACPI.",
 499         default => 1,
 500     },
 501     agent => {
 502         optional => 1,
 503         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 504         type => 'string',
 505         format => $agent_fmt,
 506     },
 507     kvm => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable KVM hardware virtualization.",
 511         default => 1,
 512     },
 513     tdf => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable time drift fix.",
 517         default => 0,
 518     },
 519     localtime => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 523             ." the `ostype` indicates a Microsoft Windows OS.",
 524     },
 525     freeze => {
 526         optional => 1,
 527         type => 'boolean',
 528         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 529     },
 530     vga => {
 531         optional => 1,
 532         type => 'string', format => $vga_fmt,
 533         description => "Configure the VGA hardware.",
 534         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 535             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 536             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 537             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 538             ." display server. For win* OS you can select how many independent displays you want,"
 539             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 540             ." using a serial device as terminal.",
 541     },
 542     watchdog => {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-watchdog',
 545         description => "Create a virtual hardware watchdog device.",
 546         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 547             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 548             ." else the watchdog will reset the guest (or execute the respective action specified)",
 549     },
 550     startdate => {
 551         optional => 1,
 552         type => 'string',
 553         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 554         description => "Set the initial date of the real time clock. Valid format for date are:"
 555             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 556         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 557         default => 'now',
 558     },
 559     startup =>  get_standard_option('pve-startup-order'),
 560     template => {
 561         optional => 1,
 562         type => 'boolean',
 563         description => "Enable/disable Template.",
 564         default => 0,
 565     },
 566     args => {
 567         optional => 1,
 568         type => 'string',
 569         description => "Arbitrary arguments passed to kvm.",
 570         verbose_description => <<EODESCR,
 571 Arbitrary arguments passed to kvm, for example:
 572
 573 args: -no-reboot -smbios 'type=0,vendor=FOO'
 574
 575 NOTE: this option is for experts only.
 576 EODESCR
 577     },
 578     tablet => {
 579         optional => 1,
 580         type => 'boolean',
 581         default => 1,
 582         description => "Enable/disable the USB tablet device.",
 583         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 584             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 585             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 586             ." may consider disabling this to save some context switches. This is turned off by"
 587             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 588     },
 589     migrate_speed => {
 590         optional => 1,
 591         type => 'integer',
 592         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 593         minimum => 0,
 594         default => 0,
 595     },
 596     migrate_downtime => {
 597         optional => 1,
 598         type => 'number',
 599         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 600         minimum => 0,
 601         default => 0.1,
 602     },
 603     cdrom => {
 604         optional => 1,
 605         type => 'string', format => 'pve-qm-ide',
 606         typetext => '<volume>',
 607         description => "This is an alias for option -ide2",
 608     },
 609     cpu => {
 610         optional => 1,
 611         description => "Emulated CPU type.",
 612         type => 'string',
 613         format => 'pve-vm-cpu-conf',
 614     },
 615     parent => get_standard_option('pve-snapshot-name', {
 616         optional => 1,
 617         description => "Parent snapshot name. This is used internally, and should not be modified.",
 618     }),
 619     snaptime => {
 620         optional => 1,
 621         description => "Timestamp for snapshots.",
 622         type => 'integer',
 623         minimum => 0,
 624     },
 625     vmstate => {
 626         optional => 1,
 627         type => 'string', format => 'pve-volume-id',
 628         description => "Reference to a volume which stores the VM state. This is used internally"
 629             ." for snapshots.",
 630     },
 631     vmstatestorage => get_standard_option('pve-storage-id', {
 632         description => "Default storage for VM state volumes/files.",
 633         optional => 1,
 634     }),
 635     runningmachine => get_standard_option('pve-qemu-machine', {
 636         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 637             ." for snapshots.",
 638     }),
 639     runningcpu => {
 640         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 641             ." internally for snapshots.",
 642         optional => 1,
 643         type => 'string',
 644         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 645         format_description => 'QEMU -cpu parameter'
 646     },
 647     machine => get_standard_option('pve-qemu-machine'),
 648     arch => {
 649         description => "Virtual processor architecture. Defaults to the host.",
 650         optional => 1,
 651         type => 'string',
 652         enum => [qw(x86_64 aarch64)],
 653     },
 654     smbios1 => {
 655         description => "Specify SMBIOS type 1 fields.",
 656         type => 'string', format => 'pve-qm-smbios1',
 657         maxLength => 512,
 658         optional => 1,
 659     },
 660     protection => {
 661         optional => 1,
 662         type => 'boolean',
 663         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 664             ." remove disk operations.",
 665         default => 0,
 666     },
 667     bios => {
 668         optional => 1,
 669         type => 'string',
 670         enum => [ qw(seabios ovmf) ],
 671         description => "Select BIOS implementation.",
 672         default => 'seabios',
 673     },
 674     vmgenid => {
 675         type => 'string',
 676         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 677         format_description => 'UUID',
 678         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 679             ." to disable explicitly.",
 680         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 681             ." value identifier to the guest OS. This allows to notify the guest operating system"
 682             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 683             ." execution or creation from a template). The guest operating system notices the"
 684             ." change, and is then able to react as appropriate by marking its copies of"
 685             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 686             ."Note that auto-creation only works when done through API/CLI create or update methods"
 687             .", but not when manually editing the config file.",
 688         default => "1 (autogenerated)",
 689         optional => 1,
 690     },
 691     hookscript => {
 692         type => 'string',
 693         format => 'pve-volume-id',
 694         optional => 1,
 695         description => "Script that will be executed during various steps in the vms lifetime.",
 696     },
 697     ivshmem => {
 698         type => 'string',
 699         format => $ivshmem_fmt,
 700         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 701             ." the host.",
 702         optional => 1,
 703     },
 704     audio0 => {
 705         type => 'string',
 706         format => $audio_fmt,
 707         description => "Configure a audio device, useful in combination with QXL/Spice.",
 708         optional => 1
 709     },
 710     spice_enhancements => {
 711         type => 'string',
 712         format => $spice_enhancements_fmt,
 713         description => "Configure additional enhancements for SPICE.",
 714         optional => 1
 715     },
 716     tags => {
 717         type => 'string', format => 'pve-tag-list',
 718         description => 'Tags of the VM. This is only meta information.',
 719         optional => 1,
 720     },
 721     rng0 => {
 722         type => 'string',
 723         format => $rng_fmt,
 724         description => "Configure a VirtIO-based Random Number Generator.",
 725         optional => 1,
 726     },
 727     meta => {
 728         type => 'string',
 729         format => $meta_info_fmt,
 730         description => "Some (read-only) meta-information about this guest.",
 731         optional => 1,
 732     },
 733     affinity => {
 734         type => 'string', format => 'pve-cpuset',
 735         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 736         optional => 1,
 737     },
 738 };
 739
 740 my $cicustom_fmt = {
 741     meta => {
 742         type => 'string',
 743         optional => 1,
 744         description => 'Specify a custom file containing all meta data passed to the VM via"
 745             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 746         format => 'pve-volume-id',
 747         format_description => 'volume',
 748     },
 749     network => {
 750         type => 'string',
 751         optional => 1,
 752         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 753         format => 'pve-volume-id',
 754         format_description => 'volume',
 755     },
 756     user => {
 757         type => 'string',
 758         optional => 1,
 759         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 760         format => 'pve-volume-id',
 761         format_description => 'volume',
 762     },
 763     vendor => {
 764         type => 'string',
 765         optional => 1,
 766         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 767         format => 'pve-volume-id',
 768         format_description => 'volume',
 769     },
 770 };
 771 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 772
 773 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 774 my $confdesc_cloudinit = {
 775     citype => {
 776         optional => 1,
 777         type => 'string',
 778         description => 'Specifies the cloud-init configuration format. The default depends on the'
 779             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 780             .' and `configdrive2` for windows.',
 781         enum => ['configdrive2', 'nocloud', 'opennebula'],
 782     },
 783     ciuser => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: User name to change ssh keys and password for instead of the"
 787             ." image's configured default user.",
 788     },
 789     cipassword => {
 790         optional => 1,
 791         type => 'string',
 792         description => 'cloud-init: Password to assign the user. Using this is generally not'
 793             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 794             .' support hashed passwords.',
 795     },
 796     ciupgrade => {
 797         optional => 1,
 798         type => 'boolean',
 799         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 800         default => 1,
 801     },
 802     cicustom => {
 803         optional => 1,
 804         type => 'string',
 805         description => 'cloud-init: Specify custom files to replace the automatically generated'
 806             .' ones at start.',
 807         format => 'pve-qm-cicustom',
 808     },
 809     searchdomain => {
 810         optional => 1,
 811         type => 'string',
 812         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 813             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 814             .' are set.',
 815     },
 816     nameserver => {
 817         optional => 1,
 818         type => 'string', format => 'address-list',
 819         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 820             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 821             .' are set.',
 822     },
 823     sshkeys => {
 824         optional => 1,
 825         type => 'string',
 826         format => 'urlencoded',
 827         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 828     },
 829 };
 830
 831 # what about other qemu settings ?
 832 #cpu => 'string',
 833 #machine => 'string',
 834 #fda => 'file',
 835 #fdb => 'file',
 836 #mtdblock => 'file',
 837 #sd => 'file',
 838 #pflash => 'file',
 839 #snapshot => 'bool',
 840 #bootp => 'file',
 841 ##tftp => 'dir',
 842 ##smb => 'dir',
 843 #kernel => 'file',
 844 #append => 'string',
 845 #initrd => 'file',
 846 ##soundhw => 'string',
 847
 848 while (my ($k, $v) = each %$confdesc) {
 849     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 850 }
 851
 852 my $MAX_NETS = 32;
 853 my $MAX_SERIAL_PORTS = 4;
 854 my $MAX_PARALLEL_PORTS = 3;
 855
 856 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 857     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 858 }
 859
 860 my $nic_model_list = [
 861     'e1000',
 862     'e1000-82540em',
 863     'e1000-82544gc',
 864     'e1000-82545em',
 865     'e1000e',
 866     'i82551',
 867     'i82557b',
 868     'i82559er',
 869     'ne2k_isa',
 870     'ne2k_pci',
 871     'pcnet',
 872     'rtl8139',
 873     'virtio',
 874     'vmxnet3',
 875 ];
 876 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 877
 878 my $net_fmt_bridge_descr = <<__EOD__;
 879 Bridge to attach the network device to. The Proxmox VE standard bridge
 880 is called 'vmbr0'.
 881
 882 If you do not specify a bridge, we create a kvm user (NATed) network
 883 device, which provides DHCP and DNS services. The following addresses
 884 are used:
 885
 886  10.0.2.2   Gateway
 887  10.0.2.3   DNS Server
 888  10.0.2.4   SMB Server
 889
 890 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 891 __EOD__
 892
 893 my $net_fmt = {
 894     macaddr  => get_standard_option('mac-addr', {
 895         description => "MAC address. That address must be unique withing your network. This is"
 896             ." automatically generated if not specified.",
 897     }),
 898     model => {
 899         type => 'string',
 900         description => "Network Card Model. The 'virtio' model provides the best performance with"
 901             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 902             ." best to use 'e1000'.",
 903         enum => $nic_model_list,
 904         default_key => 1,
 905     },
 906     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 907     bridge => get_standard_option('pve-bridge-id', {
 908         description => $net_fmt_bridge_descr,
 909         optional => 1,
 910     }),
 911     queues => {
 912         type => 'integer',
 913         minimum => 0, maximum => 64,
 914         description => 'Number of packet queues to be used on the device.',
 915         optional => 1,
 916     },
 917     rate => {
 918         type => 'number',
 919         minimum => 0,
 920         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 921         optional => 1,
 922     },
 923     tag => {
 924         type => 'integer',
 925         minimum => 1, maximum => 4094,
 926         description => 'VLAN tag to apply to packets on this interface.',
 927         optional => 1,
 928     },
 929     trunks => {
 930         type => 'string',
 931         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 932         description => 'VLAN trunks to pass through this interface.',
 933         format_description => 'vlanid[;vlanid...]',
 934         optional => 1,
 935     },
 936     firewall => {
 937         type => 'boolean',
 938         description => 'Whether this interface should be protected by the firewall.',
 939         optional => 1,
 940     },
 941     link_down => {
 942         type => 'boolean',
 943         description => 'Whether this interface should be disconnected (like pulling the plug).',
 944         optional => 1,
 945     },
 946     mtu => {
 947         type => 'integer',
 948         minimum => 1, maximum => 65520,
 949         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 950         optional => 1,
 951     },
 952 };
 953
 954 my $netdesc = {
 955     optional => 1,
 956     type => 'string', format => $net_fmt,
 957     description => "Specify network devices.",
 958 };
 959
 960 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 961
 962 my $ipconfig_fmt = {
 963     ip => {
 964         type => 'string',
 965         format => 'pve-ipv4-config',
 966         format_description => 'IPv4Format/CIDR',
 967         description => 'IPv4 address in CIDR format.',
 968         optional => 1,
 969         default => 'dhcp',
 970     },
 971     gw => {
 972         type => 'string',
 973         format => 'ipv4',
 974         format_description => 'GatewayIPv4',
 975         description => 'Default gateway for IPv4 traffic.',
 976         optional => 1,
 977         requires => 'ip',
 978     },
 979     ip6 => {
 980         type => 'string',
 981         format => 'pve-ipv6-config',
 982         format_description => 'IPv6Format/CIDR',
 983         description => 'IPv6 address in CIDR format.',
 984         optional => 1,
 985         default => 'dhcp',
 986     },
 987     gw6 => {
 988         type => 'string',
 989         format => 'ipv6',
 990         format_description => 'GatewayIPv6',
 991         description => 'Default gateway for IPv6 traffic.',
 992         optional => 1,
 993         requires => 'ip6',
 994     },
 995 };
 996 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 997 my $ipconfigdesc = {
 998     optional => 1,
 999     type => 'string', format => 'pve-qm-ipconfig',
1000     description => <<'EODESCR',
1001 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1002
1003 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1004
1005 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1006 gateway should be provided.
1007 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1008 cloud-init 19.4 or newer.
1009
1010 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1011 dhcp on IPv4.
1012 EODESCR
1013 };
1014 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1015
1016 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1017     $confdesc->{"net$i"} = $netdesc;
1018     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1019 }
1020
1021 foreach my $key (keys %$confdesc_cloudinit) {
1022     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1023 }
1024
1025 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1026 sub pve_verify_cpuset {
1027     my ($set_text, $noerr) = @_;
1028
1029     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1030
1031     if ($@) {
1032         return if $noerr;
1033         die "unable to parse cpuset option\n";
1034     }
1035
1036     return PVE::CpuSet->new($members)->short_string();
1037 }
1038
1039 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1040 sub verify_volume_id_or_qm_path {
1041     my ($volid, $noerr) = @_;
1042
1043     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1044
1045     return verify_volume_id_or_absolute_path($volid, $noerr);
1046 }
1047
1048 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1049 sub verify_volume_id_or_absolute_path {
1050     my ($volid, $noerr) = @_;
1051
1052     return $volid if $volid =~ m|^/|;
1053
1054     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1055     if ($@) {
1056         return if $noerr;
1057         die $@;
1058     }
1059     return $volid;
1060 }
1061
1062 my $serialdesc = {
1063         optional => 1,
1064         type => 'string',
1065         pattern => '(/dev/.+|socket)',
1066         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1067         verbose_description =>  <<EODESCR,
1068 Create a serial device inside the VM (n is 0 to 3), and pass through a
1069 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1070 host side (use 'qm terminal' to open a terminal connection).
1071
1072 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1073 use with special care.
1074
1075 CAUTION: Experimental! User reported problems with this option.
1076 EODESCR
1077 };
1078
1079 my $paralleldesc= {
1080         optional => 1,
1081         type => 'string',
1082         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1083         description =>  "Map host parallel devices (n is 0 to 2).",
1084         verbose_description =>  <<EODESCR,
1085 Map host parallel devices (n is 0 to 2).
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1095     $confdesc->{"parallel$i"} = $paralleldesc;
1096 }
1097
1098 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1099     $confdesc->{"serial$i"} = $serialdesc;
1100 }
1101
1102 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1103     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1104 }
1105
1106 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1107     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1108 }
1109
1110 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1111     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1112 }
1113
1114 my $boot_fmt = {
1115     legacy => {
1116         optional => 1,
1117         default_key => 1,
1118         type => 'string',
1119         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1120                      . " Deprecated, use 'order=' instead.",
1121         pattern => '[acdn]{1,4}',
1122         format_description => "[acdn]{1,4}",
1123
1124         # note: this is also the fallback if boot: is not given at all
1125         default => 'cdn',
1126     },
1127     order => {
1128         optional => 1,
1129         type => 'string',
1130         format => 'pve-qm-bootdev-list',
1131         format_description => "device[;device...]",
1132         description => <<EODESC,
1133 The guest will attempt to boot from devices in the order they appear here.
1134
1135 Disks, optical drives and passed-through storage USB devices will be directly
1136 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1137 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1138
1139 Note that only devices in this list will be marked as bootable and thus loaded
1140 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1141 (e.g. software-raid), you need to specify all of them here.
1142
1143 Overrides the deprecated 'legacy=[acdn]*' value when given.
1144 EODESC
1145     },
1146 };
1147 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1148
1149 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1150 sub verify_bootdev {
1151     my ($dev, $noerr) = @_;
1152
1153     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1154     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1155
1156     my $check = sub {
1157         my ($base) = @_;
1158         return 0 if $dev !~ m/^$base\d+$/;
1159         return 0 if !$confdesc->{$dev};
1160         return 1;
1161     };
1162
1163     return $dev if $check->("net");
1164     return $dev if $check->("usb");
1165     return $dev if $check->("hostpci");
1166
1167     return if $noerr;
1168     die "invalid boot device '$dev'\n";
1169 }
1170
1171 sub print_bootorder {
1172     my ($devs) = @_;
1173     return "" if !@$devs;
1174     my $data = { order => join(';', @$devs) };
1175     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1176 }
1177
1178 my $kvm_api_version = 0;
1179
1180 sub kvm_version {
1181     return $kvm_api_version if $kvm_api_version;
1182
1183     open my $fh, '<', '/dev/kvm' or return;
1184
1185     # 0xae00 => KVM_GET_API_VERSION
1186     $kvm_api_version = ioctl($fh, 0xae00, 0);
1187     close($fh);
1188
1189     return $kvm_api_version;
1190 }
1191
1192 my $kvm_user_version = {};
1193 my $kvm_mtime = {};
1194
1195 sub kvm_user_version {
1196     my ($binary) = @_;
1197
1198     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1199     my $st = stat($binary);
1200
1201     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1202     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1203         $cachedmtime == $st->mtime;
1204
1205     $kvm_user_version->{$binary} = 'unknown';
1206     $kvm_mtime->{$binary} = $st->mtime;
1207
1208     my $code = sub {
1209         my $line = shift;
1210         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1211             $kvm_user_version->{$binary} = $2;
1212         }
1213     };
1214
1215     eval { run_command([$binary, '--version'], outfunc => $code); };
1216     warn $@ if $@;
1217
1218     return $kvm_user_version->{$binary};
1219
1220 }
1221 my sub extract_version {
1222     my ($machine_type, $version) = @_;
1223     $version = kvm_user_version() if !defined($version);
1224     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1225 }
1226
1227 sub kernel_has_vhost_net {
1228     return -c '/dev/vhost-net';
1229 }
1230
1231 sub option_exists {
1232     my $key = shift;
1233     return defined($confdesc->{$key});
1234 }
1235
1236 my $cdrom_path;
1237 sub get_cdrom_path {
1238
1239     return $cdrom_path if defined($cdrom_path);
1240
1241     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1242
1243     if (!defined($cdrom_path)) {
1244         log_warn("no physical CD-ROM available, ignoring");
1245         $cdrom_path = '';
1246     }
1247
1248     return $cdrom_path;
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269      if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1314             if !$vtype;
1315         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1316         verify_media_type($opt, $vtype, $drive->{media});
1317         $drive->{file} = $volid;
1318     }
1319
1320     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1321 }
1322
1323 sub parse_hotplug_features {
1324     my ($data) = @_;
1325
1326     my $res = {};
1327
1328     return $res if $data eq '0';
1329
1330     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1331
1332     foreach my $feature (PVE::Tools::split_list($data)) {
1333         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1334             $res->{$1} = 1;
1335         } else {
1336             die "invalid hotplug feature '$feature'\n";
1337         }
1338     }
1339     return $res;
1340 }
1341
1342 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1343 sub pve_verify_hotplug_features {
1344     my ($value, $noerr) = @_;
1345
1346     return $value if parse_hotplug_features($value);
1347
1348     return if $noerr;
1349
1350     die "unable to parse hotplug option\n";
1351 }
1352
1353 sub assert_clipboard_config {
1354     my ($vga) = @_;
1355
1356     my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1357
1358     if (
1359         $vga->{'clipboard'}
1360         && $vga->{'clipboard'} eq 'vnc'
1361         && $vga->{type}
1362         && $vga->{type} !~ $clipboard_regex
1363     ) {
1364         die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1365     }
1366 }
1367
1368 sub print_tabletdevice_full {
1369     my ($conf, $arch) = @_;
1370
1371     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1372
1373     # we use uhci for old VMs because tablet driver was buggy in older qemu
1374     my $usbbus;
1375     if ($q35 || $arch eq 'aarch64') {
1376         $usbbus = 'ehci';
1377     } else {
1378         $usbbus = 'uhci';
1379     }
1380
1381     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1382 }
1383
1384 sub print_keyboarddevice_full {
1385     my ($conf, $arch) = @_;
1386
1387     return if $arch ne 'aarch64';
1388
1389     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1390 }
1391
1392 my sub get_drive_id {
1393     my ($drive) = @_;
1394     return "$drive->{interface}$drive->{index}";
1395 }
1396
1397 sub print_drivedevice_full {
1398     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1399
1400     my $device = '';
1401     my $maxdev = 0;
1402
1403     my $drive_id = get_drive_id($drive);
1404     if ($drive->{interface} eq 'virtio') {
1405         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1406         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1407         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1408     } elsif ($drive->{interface} eq 'scsi') {
1409
1410         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1411         my $unit = $drive->{index} % $maxdev;
1412
1413         my $machine_version = extract_version($machine_type, kvm_user_version());
1414         my $devicetype  = PVE::QemuServer::Drive::get_scsi_devicetype(
1415             $drive, $storecfg, $machine_version);
1416
1417         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1418             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1419         } else {
1420             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1421                 .",lun=$drive->{index}";
1422         }
1423         $device .= ",drive=drive-$drive_id,id=$drive_id";
1424
1425         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1426             $device .= ",rotation_rate=1";
1427         }
1428         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1429
1430         # only scsi-hd and scsi-cd support passing vendor and product information
1431         if ($devicetype eq 'hd' || $devicetype eq 'cd') {
1432             if (my $vendor = $drive->{vendor}) {
1433                 $device .= ",vendor=$vendor";
1434             }
1435             if (my $product = $drive->{product}) {
1436                 $device .= ",product=$product";
1437             }
1438         }
1439
1440     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1441         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1442         my $controller = int($drive->{index} / $maxdev);
1443         my $unit = $drive->{index} % $maxdev;
1444
1445         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1446         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1447         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1448         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1449         # were before. Move odd ones up by 2 where they don't clash.
1450         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1451             $controller += 2 * ($unit % 2);
1452             $unit = 0;
1453         }
1454
1455         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1456
1457         $device = "ide-$devicetype";
1458         if ($drive->{interface} eq 'ide') {
1459             $device .= ",bus=ide.$controller,unit=$unit";
1460         } else {
1461             $device .= ",bus=ahci$controller.$unit";
1462         }
1463         $device .= ",drive=drive-$drive_id,id=$drive_id";
1464
1465         if ($devicetype eq 'hd') {
1466             if (my $model = $drive->{model}) {
1467                 $model = URI::Escape::uri_unescape($model);
1468                 $device .= ",model=$model";
1469             }
1470             if ($drive->{ssd}) {
1471                 $device .= ",rotation_rate=1";
1472             }
1473         }
1474         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1475     } elsif ($drive->{interface} eq 'usb') {
1476         die "implement me";
1477         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1478     } else {
1479         die "unsupported interface type";
1480     }
1481
1482     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1483
1484     if (my $serial = $drive->{serial}) {
1485         $serial = URI::Escape::uri_unescape($serial);
1486         $device .= ",serial=$serial";
1487     }
1488
1489
1490     return $device;
1491 }
1492
1493 sub get_initiator_name {
1494     my $initiator;
1495
1496     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1497     while (defined(my $line = <$fh>)) {
1498         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1499         $initiator = $1;
1500         last;
1501     }
1502     $fh->close();
1503
1504     return $initiator;
1505 }
1506
1507 my sub storage_allows_io_uring_default {
1508     my ($scfg, $cache_direct) = @_;
1509
1510     # io_uring with cache mode writeback or writethrough on krbd will hang...
1511     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1512
1513     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1514     # sometimes, just plain disable...
1515     return if $scfg && $scfg->{type} eq 'lvm';
1516
1517     # io_uring causes problems when used with CIFS since kernel 5.15
1518     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1519     return if $scfg && $scfg->{type} eq 'cifs';
1520
1521     return 1;
1522 }
1523
1524 my sub drive_uses_cache_direct {
1525     my ($drive, $scfg) = @_;
1526
1527     my $cache_direct = 0;
1528
1529     if (my $cache = $drive->{cache}) {
1530         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1531     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1532         $cache_direct = 1;
1533     }
1534
1535     return $cache_direct;
1536 }
1537
1538 sub print_drive_commandline_full {
1539     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1540
1541     my $path;
1542     my $volid = $drive->{file};
1543     my $format = $drive->{format};
1544     my $drive_id = get_drive_id($drive);
1545
1546     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1547     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1548
1549     if (drive_is_cdrom($drive)) {
1550         $path = get_iso_path($storecfg, $vmid, $volid);
1551         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1552     } else {
1553         if ($storeid) {
1554             $path = PVE::Storage::path($storecfg, $volid);
1555             $format //= qemu_img_format($scfg, $volname);
1556         } else {
1557             $path = $volid;
1558             $format //= "raw";
1559         }
1560    }
1561
1562    my $is_rbd = $path =~ m/^rbd:/;
1563
1564     my $opts = '';
1565     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1566     foreach my $o (@qemu_drive_options) {
1567         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1568     }
1569
1570     # snapshot only accepts on|off
1571     if (defined($drive->{snapshot})) {
1572         my $v = $drive->{snapshot} ? 'on' : 'off';
1573         $opts .= ",snapshot=$v";
1574     }
1575
1576     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1577         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1578     }
1579
1580     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1581         my ($dir, $qmpname) = @$type;
1582         if (my $v = $drive->{"mbps$dir"}) {
1583             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1584         }
1585         if (my $v = $drive->{"mbps${dir}_max"}) {
1586             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1587         }
1588         if (my $v = $drive->{"bps${dir}_max_length"}) {
1589             $opts .= ",throttling.bps$qmpname-max-length=$v";
1590         }
1591         if (my $v = $drive->{"iops${dir}"}) {
1592             $opts .= ",throttling.iops$qmpname=$v";
1593         }
1594         if (my $v = $drive->{"iops${dir}_max"}) {
1595             $opts .= ",throttling.iops$qmpname-max=$v";
1596         }
1597         if (my $v = $drive->{"iops${dir}_max_length"}) {
1598             $opts .= ",throttling.iops$qmpname-max-length=$v";
1599         }
1600     }
1601
1602     if ($pbs_name) {
1603         $format = "rbd" if $is_rbd;
1604         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1605             if !$format;
1606         $opts .= ",format=alloc-track,file.driver=$format";
1607     } elsif ($format) {
1608         $opts .= ",format=$format";
1609     }
1610
1611     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1612
1613     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1614
1615     if (!$drive->{aio}) {
1616         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1617             # io_uring supports all cache modes
1618             $opts .= ",aio=io_uring";
1619         } else {
1620             # aio native works only with O_DIRECT
1621             if($cache_direct) {
1622                 $opts .= ",aio=native";
1623             } else {
1624                 $opts .= ",aio=threads";
1625             }
1626         }
1627     }
1628
1629     if (!drive_is_cdrom($drive)) {
1630         my $detectzeroes;
1631         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1632             $detectzeroes = 'off';
1633         } elsif ($drive->{discard}) {
1634             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1635         } else {
1636             # This used to be our default with discard not being specified:
1637             $detectzeroes = 'on';
1638         }
1639
1640         # note: 'detect-zeroes' works per blockdev and we want it to persist
1641         # after the alloc-track is removed, so put it on 'file' directly
1642         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1643         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1644     }
1645
1646     if ($pbs_name) {
1647         $opts .= ",backing=$pbs_name";
1648         $opts .= ",auto-remove=on";
1649     }
1650
1651     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1652     my $file_param = "file";
1653     if ($pbs_name) {
1654         # non-rbd drivers require the underlying file to be a seperate block
1655         # node, so add a second .file indirection
1656         $file_param .= ".file" if !$is_rbd;
1657         $file_param .= ".filename";
1658     }
1659     my $pathinfo = $path ? "$file_param=$path," : '';
1660
1661     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1662 }
1663
1664 sub print_pbs_blockdev {
1665     my ($pbs_conf, $pbs_name) = @_;
1666     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1667     $blockdev .= ",repository=$pbs_conf->{repository}";
1668     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1669     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1670     $blockdev .= ",archive=$pbs_conf->{archive}";
1671     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1672     return $blockdev;
1673 }
1674
1675 sub print_netdevice_full {
1676     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1677
1678     my $device = $net->{model};
1679     if ($net->{model} eq 'virtio') {
1680          $device = 'virtio-net-pci';
1681      };
1682
1683     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1684     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1685     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1686         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1687         # and out of each queue plus one config interrupt and control vector queue
1688         my $vectors = $net->{queues} * 2 + 2;
1689         $tmpstr .= ",vectors=$vectors,mq=on";
1690         if (min_version($machine_version, 7, 1)) {
1691             $tmpstr .= ",packed=on";
1692         }
1693     }
1694
1695     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1696         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1697     }
1698
1699     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1700
1701     if (my $mtu = $net->{mtu}) {
1702         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1703             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1704             if ($mtu == 1) {
1705                  $mtu = $bridge_mtu;
1706             } elsif ($mtu < 576) {
1707                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1708             } elsif ($mtu > $bridge_mtu) {
1709                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1710             }
1711             $tmpstr .= ",host_mtu=$mtu";
1712         } else {
1713             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1714         }
1715     }
1716
1717     if ($use_old_bios_files) {
1718         my $romfile;
1719         if ($device eq 'virtio-net-pci') {
1720             $romfile = 'pxe-virtio.rom';
1721         } elsif ($device eq 'e1000') {
1722             $romfile = 'pxe-e1000.rom';
1723         } elsif ($device eq 'e1000e') {
1724             $romfile = 'pxe-e1000e.rom';
1725         } elsif ($device eq 'ne2k') {
1726             $romfile = 'pxe-ne2k_pci.rom';
1727         } elsif ($device eq 'pcnet') {
1728             $romfile = 'pxe-pcnet.rom';
1729         } elsif ($device eq 'rtl8139') {
1730             $romfile = 'pxe-rtl8139.rom';
1731         }
1732         $tmpstr .= ",romfile=$romfile" if $romfile;
1733     }
1734
1735     return $tmpstr;
1736 }
1737
1738 sub print_netdev_full {
1739     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1740
1741     my $i = '';
1742     if ($netid =~ m/^net(\d+)$/) {
1743         $i = int($1);
1744     }
1745
1746     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1747
1748     my $ifname = "tap${vmid}i$i";
1749
1750     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1751     die "interface name '$ifname' is too long (max 15 character)\n"
1752         if length($ifname) >= 16;
1753
1754     my $vhostparam = '';
1755     if (is_native($arch)) {
1756         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1757     }
1758
1759     my $vmname = $conf->{name} || "vm$vmid";
1760
1761     my $netdev = "";
1762     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1763
1764     if ($net->{bridge}) {
1765         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1766             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1767     } else {
1768         $netdev = "type=user,id=$netid,hostname=$vmname";
1769     }
1770
1771     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1772
1773     return $netdev;
1774 }
1775
1776 my $vga_map = {
1777     'cirrus' => 'cirrus-vga',
1778     'std' => 'VGA',
1779     'vmware' => 'vmware-svga',
1780     'virtio' => 'virtio-vga',
1781     'virtio-gl' => 'virtio-vga-gl',
1782 };
1783
1784 sub print_vga_device {
1785     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1786
1787     my $type = $vga_map->{$vga->{type}};
1788     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1789         $type = 'virtio-gpu';
1790     }
1791     my $vgamem_mb = $vga->{memory};
1792
1793     my $max_outputs = '';
1794     if ($qxlnum) {
1795         $type = $id ? 'qxl' : 'qxl-vga';
1796
1797         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1798             # set max outputs so linux can have up to 4 qxl displays with one device
1799             if (min_version($machine_version, 4, 1)) {
1800                 $max_outputs = ",max_outputs=4";
1801             }
1802         }
1803     }
1804
1805     die "no devicetype for $vga->{type}\n" if !$type;
1806
1807     my $memory = "";
1808     if ($vgamem_mb) {
1809         if ($vga->{type} =~ /^virtio/) {
1810             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1811             $memory = ",max_hostmem=$bytes";
1812         } elsif ($qxlnum) {
1813             # from https://www.spice-space.org/multiple-monitors.html
1814             $memory = ",vgamem_mb=$vga->{memory}";
1815             my $ram = $vgamem_mb * 4;
1816             my $vram = $vgamem_mb * 2;
1817             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1818         } else {
1819             $memory = ",vgamem_mb=$vga->{memory}";
1820         }
1821     } elsif ($qxlnum && $id) {
1822         $memory = ",ram_size=67108864,vram_size=33554432";
1823     }
1824
1825     my $edidoff = "";
1826     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1827         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1828     }
1829
1830     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1831     my $vgaid = "vga" . ($id // '');
1832     my $pciaddr;
1833     if ($q35 && $vgaid eq 'vga') {
1834         # the first display uses pcie.0 bus on q35 machines
1835         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1836     } else {
1837         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1838     }
1839
1840     if ($vga->{type} eq 'virtio-gl') {
1841         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1842         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1843             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1844
1845         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1846             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1847     }
1848
1849     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1850 }
1851
1852 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1853 sub parse_net {
1854     my ($data, $disable_mac_autogen) = @_;
1855
1856     my $res = eval { parse_property_string($net_fmt, $data) };
1857     if ($@) {
1858         warn $@;
1859         return;
1860     }
1861     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1862         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1863         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1864     }
1865     return $res;
1866 }
1867
1868 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1869 sub parse_ipconfig {
1870     my ($data) = @_;
1871
1872     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1873     if ($@) {
1874         warn $@;
1875         return;
1876     }
1877
1878     if ($res->{gw} && !$res->{ip}) {
1879         warn 'gateway specified without specifying an IP address';
1880         return;
1881     }
1882     if ($res->{gw6} && !$res->{ip6}) {
1883         warn 'IPv6 gateway specified without specifying an IPv6 address';
1884         return;
1885     }
1886     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1887         warn 'gateway specified together with DHCP';
1888         return;
1889     }
1890     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1891         # gw6 + auto/dhcp
1892         warn "IPv6 gateway specified together with $res->{ip6} address";
1893         return;
1894     }
1895
1896     if (!$res->{ip} && !$res->{ip6}) {
1897         return { ip => 'dhcp', ip6 => 'dhcp' };
1898     }
1899
1900     return $res;
1901 }
1902
1903 sub print_net {
1904     my $net = shift;
1905
1906     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1907 }
1908
1909 sub add_random_macs {
1910     my ($settings) = @_;
1911
1912     foreach my $opt (keys %$settings) {
1913         next if $opt !~ m/^net(\d+)$/;
1914         my $net = parse_net($settings->{$opt});
1915         next if !$net;
1916         $settings->{$opt} = print_net($net);
1917     }
1918 }
1919
1920 sub vm_is_volid_owner {
1921     my ($storecfg, $vmid, $volid) = @_;
1922
1923     if ($volid !~  m|^/|) {
1924         my ($path, $owner);
1925         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1926         if ($owner && ($owner == $vmid)) {
1927             return 1;
1928         }
1929     }
1930
1931     return;
1932 }
1933
1934 sub vmconfig_register_unused_drive {
1935     my ($storecfg, $vmid, $conf, $drive) = @_;
1936
1937     if (drive_is_cloudinit($drive)) {
1938         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1939         warn $@ if $@;
1940         delete $conf->{cloudinit};
1941     } elsif (!drive_is_cdrom($drive)) {
1942         my $volid = $drive->{file};
1943         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1944             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1945         }
1946     }
1947 }
1948
1949 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1950 my $smbios1_fmt = {
1951     uuid => {
1952         type => 'string',
1953         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1954         format_description => 'UUID',
1955         description => "Set SMBIOS1 UUID.",
1956         optional => 1,
1957     },
1958     version => {
1959         type => 'string',
1960         pattern => '[A-Za-z0-9+\/]+={0,2}',
1961         format_description => 'Base64 encoded string',
1962         description => "Set SMBIOS1 version.",
1963         optional => 1,
1964     },
1965     serial => {
1966         type => 'string',
1967         pattern => '[A-Za-z0-9+\/]+={0,2}',
1968         format_description => 'Base64 encoded string',
1969         description => "Set SMBIOS1 serial number.",
1970         optional => 1,
1971     },
1972     manufacturer => {
1973         type => 'string',
1974         pattern => '[A-Za-z0-9+\/]+={0,2}',
1975         format_description => 'Base64 encoded string',
1976         description => "Set SMBIOS1 manufacturer.",
1977         optional => 1,
1978     },
1979     product => {
1980         type => 'string',
1981         pattern => '[A-Za-z0-9+\/]+={0,2}',
1982         format_description => 'Base64 encoded string',
1983         description => "Set SMBIOS1 product ID.",
1984         optional => 1,
1985     },
1986     sku => {
1987         type => 'string',
1988         pattern => '[A-Za-z0-9+\/]+={0,2}',
1989         format_description => 'Base64 encoded string',
1990         description => "Set SMBIOS1 SKU string.",
1991         optional => 1,
1992     },
1993     family => {
1994         type => 'string',
1995         pattern => '[A-Za-z0-9+\/]+={0,2}',
1996         format_description => 'Base64 encoded string',
1997         description => "Set SMBIOS1 family string.",
1998         optional => 1,
1999     },
2000     base64 => {
2001         type => 'boolean',
2002         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2003         optional => 1,
2004     },
2005 };
2006
2007 sub parse_smbios1 {
2008     my ($data) = @_;
2009
2010     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2011     warn $@ if $@;
2012     return $res;
2013 }
2014
2015 sub print_smbios1 {
2016     my ($smbios1) = @_;
2017     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2018 }
2019
2020 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2021
2022 sub parse_watchdog {
2023     my ($value) = @_;
2024
2025     return if !$value;
2026
2027     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2028     warn $@ if $@;
2029     return $res;
2030 }
2031
2032 sub parse_guest_agent {
2033     my ($conf) = @_;
2034
2035     return {} if !defined($conf->{agent});
2036
2037     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2038     warn $@ if $@;
2039
2040     # if the agent is disabled ignore the other potentially set properties
2041     return {} if !$res->{enabled};
2042     return $res;
2043 }
2044
2045 sub get_qga_key {
2046     my ($conf, $key) = @_;
2047     return undef if !defined($conf->{agent});
2048
2049     my $agent = parse_guest_agent($conf);
2050     return $agent->{$key};
2051 }
2052
2053 sub parse_vga {
2054     my ($value) = @_;
2055
2056     return {} if !$value;
2057     my $res = eval { parse_property_string($vga_fmt, $value) };
2058     warn $@ if $@;
2059     return $res;
2060 }
2061
2062 sub parse_rng {
2063     my ($value) = @_;
2064
2065     return if !$value;
2066
2067     my $res = eval { parse_property_string($rng_fmt, $value) };
2068     warn $@ if $@;
2069     return $res;
2070 }
2071
2072 sub parse_meta_info {
2073     my ($value) = @_;
2074
2075     return if !$value;
2076
2077     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2078     warn $@ if $@;
2079     return $res;
2080 }
2081
2082 sub new_meta_info_string {
2083     my () = @_; # for now do not allow to override any value
2084
2085     return PVE::JSONSchema::print_property_string(
2086         {
2087             'creation-qemu' => kvm_user_version(),
2088             ctime => "". int(time()),
2089         },
2090         $meta_info_fmt
2091     );
2092 }
2093
2094 sub qemu_created_version_fixups {
2095     my ($conf, $forcemachine, $kvmver) = @_;
2096
2097     my $meta = parse_meta_info($conf->{meta}) // {};
2098     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2099
2100     # check if we need to apply some handling for VMs that always use the latest machine version but
2101     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2102     # would be required (we do not want to pin machine version for non-windows OS type)
2103     if (
2104         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2105         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2106         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2107         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2108     ) {
2109         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2110         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2111             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2112             # and thus with the predictable interface naming of systemd
2113             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2114         }
2115     }
2116     return;
2117 }
2118
2119 # add JSON properties for create and set function
2120 sub json_config_properties {
2121     my ($prop, $with_disk_alloc) = @_;
2122
2123     my $skip_json_config_opts = {
2124         parent => 1,
2125         snaptime => 1,
2126         vmstate => 1,
2127         runningmachine => 1,
2128         runningcpu => 1,
2129         meta => 1,
2130     };
2131
2132     foreach my $opt (keys %$confdesc) {
2133         next if $skip_json_config_opts->{$opt};
2134
2135         if ($with_disk_alloc && is_valid_drivename($opt)) {
2136             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2137         } else {
2138             $prop->{$opt} = $confdesc->{$opt};
2139         }
2140     }
2141
2142     return $prop;
2143 }
2144
2145 # Properties that we can read from an OVF file
2146 sub json_ovf_properties {
2147     my $prop = {};
2148
2149     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2150         $prop->{$device} = {
2151             type => 'string',
2152             format => 'pve-volume-id-or-absolute-path',
2153             description => "Disk image that gets imported to $device",
2154             optional => 1,
2155         };
2156     }
2157
2158     $prop->{cores} = {
2159         type => 'integer',
2160         description => "The number of CPU cores.",
2161         optional => 1,
2162     };
2163     $prop->{memory} = {
2164         type => 'integer',
2165         description => "Amount of RAM for the VM in MB.",
2166         optional => 1,
2167     };
2168     $prop->{name} = {
2169         type => 'string',
2170         description => "Name of the VM.",
2171         optional => 1,
2172     };
2173
2174     return $prop;
2175 }
2176
2177 # return copy of $confdesc_cloudinit to generate documentation
2178 sub cloudinit_config_properties {
2179
2180     return dclone($confdesc_cloudinit);
2181 }
2182
2183 sub cloudinit_pending_properties {
2184     my $p = {
2185         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2186         name => 1,
2187     };
2188     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2189     return $p;
2190 }
2191
2192 sub check_type {
2193     my ($key, $value) = @_;
2194
2195     die "unknown setting '$key'\n" if !$confdesc->{$key};
2196
2197     my $type = $confdesc->{$key}->{type};
2198
2199     if (!defined($value)) {
2200         die "got undefined value\n";
2201     }
2202
2203     if ($value =~ m/[\n\r]/) {
2204         die "property contains a line feed\n";
2205     }
2206
2207     if ($type eq 'boolean') {
2208         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2209         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2210         die "type check ('boolean') failed - got '$value'\n";
2211     } elsif ($type eq 'integer') {
2212         return int($1) if $value =~ m/^(\d+)$/;
2213         die "type check ('integer') failed - got '$value'\n";
2214     } elsif ($type eq 'number') {
2215         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2216         die "type check ('number') failed - got '$value'\n";
2217     } elsif ($type eq 'string') {
2218         if (my $fmt = $confdesc->{$key}->{format}) {
2219             PVE::JSONSchema::check_format($fmt, $value);
2220             return $value;
2221         }
2222         $value =~ s/^\"(.*)\"$/$1/;
2223         return $value;
2224     } else {
2225         die "internal error"
2226     }
2227 }
2228
2229 sub destroy_vm {
2230     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2231
2232     my $conf = PVE::QemuConfig->load_config($vmid);
2233
2234     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2235         PVE::QemuConfig->check_lock($conf);
2236     }
2237
2238     if ($conf->{template}) {
2239         # check if any base image is still used by a linked clone
2240         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2241                 my ($ds, $drive) = @_;
2242                 return if drive_is_cdrom($drive);
2243
2244                 my $volid = $drive->{file};
2245                 return if !$volid || $volid =~ m|^/|;
2246
2247                 die "base volume '$volid' is still in use by linked cloned\n"
2248                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2249
2250         });
2251     }
2252
2253     my $volids = {};
2254     my $remove_owned_drive = sub {
2255         my ($ds, $drive) = @_;
2256         return if drive_is_cdrom($drive, 1);
2257
2258         my $volid = $drive->{file};
2259         return if !$volid || $volid =~ m|^/|;
2260         return if $volids->{$volid};
2261
2262         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2263         return if !$path || !$owner || ($owner != $vmid);
2264
2265         $volids->{$volid} = 1;
2266         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2267         warn "Could not remove disk '$volid', check manually: $@" if $@;
2268     };
2269
2270     # only remove disks owned by this VM (referenced in the config)
2271     my $include_opts = {
2272         include_unused => 1,
2273         extra_keys => ['vmstate'],
2274     };
2275     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2276
2277     for my $snap (values %{$conf->{snapshots}}) {
2278         next if !defined($snap->{vmstate});
2279         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2280         next if !defined($drive);
2281         $remove_owned_drive->('vmstate', $drive);
2282     }
2283
2284     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2285
2286     if ($purge_unreferenced) { # also remove unreferenced disk
2287         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2288         PVE::Storage::foreach_volid($vmdisks, sub {
2289             my ($volid, $sid, $volname, $d) = @_;
2290             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2291             warn $@ if $@;
2292         });
2293     }
2294
2295     eval { delete_ifaces_ipams_ips($conf, $vmid)};
2296     warn $@ if $@;
2297
2298     if (defined $replacement_conf) {
2299         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2300     } else {
2301         PVE::QemuConfig->destroy_config($vmid);
2302     }
2303 }
2304
2305 sub parse_vm_config {
2306     my ($filename, $raw, $strict) = @_;
2307
2308     return if !defined($raw);
2309
2310     my $res = {
2311         digest => Digest::SHA::sha1_hex($raw),
2312         snapshots => {},
2313         pending => {},
2314         cloudinit => {},
2315     };
2316
2317     my $handle_error = sub {
2318         my ($msg) = @_;
2319
2320         if ($strict) {
2321             die $msg;
2322         } else {
2323             warn $msg;
2324         }
2325     };
2326
2327     $filename =~ m|/qemu-server/(\d+)\.conf$|
2328         || die "got strange filename '$filename'";
2329
2330     my $vmid = $1;
2331
2332     my $conf = $res;
2333     my $descr;
2334     my $finish_description = sub {
2335         if (defined($descr)) {
2336             $descr =~ s/\s+$//;
2337             $conf->{description} = $descr;
2338         }
2339         $descr = undef;
2340     };
2341     my $section = '';
2342
2343     my @lines = split(/\n/, $raw);
2344     foreach my $line (@lines) {
2345         next if $line =~ m/^\s*$/;
2346
2347         if ($line =~ m/^\[PENDING\]\s*$/i) {
2348             $section = 'pending';
2349             $finish_description->();
2350             $conf = $res->{$section} = {};
2351             next;
2352         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2353             $section = 'cloudinit';
2354             $finish_description->();
2355             $conf = $res->{$section} = {};
2356             next;
2357
2358         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2359             $section = $1;
2360             $finish_description->();
2361             $conf = $res->{snapshots}->{$section} = {};
2362             next;
2363         }
2364
2365         if ($line =~ m/^\#(.*)$/) {
2366             $descr = '' if !defined($descr);
2367             $descr .= PVE::Tools::decode_text($1) . "\n";
2368             next;
2369         }
2370
2371         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2372             $descr = '' if !defined($descr);
2373             $descr .= PVE::Tools::decode_text($2);
2374         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2375             $conf->{snapstate} = $1;
2376         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2377             my $key = $1;
2378             my $value = $2;
2379             $conf->{$key} = $value;
2380         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2381             my $value = $1;
2382             if ($section eq 'pending') {
2383                 $conf->{delete} = $value; # we parse this later
2384             } else {
2385                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2386             }
2387         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2388             my $key = $1;
2389             my $value = $2;
2390             if ($section eq 'cloudinit') {
2391                 # ignore validation only used for informative purpose
2392                 $conf->{$key} = $value;
2393                 next;
2394             }
2395             eval { $value = check_type($key, $value); };
2396             if ($@) {
2397                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2398             } else {
2399                 $key = 'ide2' if $key eq 'cdrom';
2400                 my $fmt = $confdesc->{$key}->{format};
2401                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2402                     my $v = parse_drive($key, $value);
2403                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2404                         $v->{file} = $volid;
2405                         $value = print_drive($v);
2406                     } else {
2407                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2408                         next;
2409                     }
2410                 }
2411
2412                 $conf->{$key} = $value;
2413             }
2414         } else {
2415             $handle_error->("vm $vmid - unable to parse config: $line\n");
2416         }
2417     }
2418
2419     $finish_description->();
2420     delete $res->{snapstate}; # just to be sure
2421
2422     return $res;
2423 }
2424
2425 sub write_vm_config {
2426     my ($filename, $conf) = @_;
2427
2428     delete $conf->{snapstate}; # just to be sure
2429
2430     if ($conf->{cdrom}) {
2431         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2432         $conf->{ide2} = $conf->{cdrom};
2433         delete $conf->{cdrom};
2434     }
2435
2436     # we do not use 'smp' any longer
2437     if ($conf->{sockets}) {
2438         delete $conf->{smp};
2439     } elsif ($conf->{smp}) {
2440         $conf->{sockets} = $conf->{smp};
2441         delete $conf->{cores};
2442         delete $conf->{smp};
2443     }
2444
2445     my $used_volids = {};
2446
2447     my $cleanup_config = sub {
2448         my ($cref, $pending, $snapname) = @_;
2449
2450         foreach my $key (keys %$cref) {
2451             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2452                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2453             my $value = $cref->{$key};
2454             if ($key eq 'delete') {
2455                 die "propertry 'delete' is only allowed in [PENDING]\n"
2456                     if !$pending;
2457                 # fixme: check syntax?
2458                 next;
2459             }
2460             eval { $value = check_type($key, $value); };
2461             die "unable to parse value of '$key' - $@" if $@;
2462
2463             $cref->{$key} = $value;
2464
2465             if (!$snapname && is_valid_drivename($key)) {
2466                 my $drive = parse_drive($key, $value);
2467                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2468             }
2469         }
2470     };
2471
2472     &$cleanup_config($conf);
2473
2474     &$cleanup_config($conf->{pending}, 1);
2475
2476     foreach my $snapname (keys %{$conf->{snapshots}}) {
2477         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2478         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2479     }
2480
2481     # remove 'unusedX' settings if we re-add a volume
2482     foreach my $key (keys %$conf) {
2483         my $value = $conf->{$key};
2484         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2485             delete $conf->{$key};
2486         }
2487     }
2488
2489     my $generate_raw_config = sub {
2490         my ($conf, $pending) = @_;
2491
2492         my $raw = '';
2493
2494         # add description as comment to top of file
2495         if (defined(my $descr = $conf->{description})) {
2496             if ($descr) {
2497                 foreach my $cl (split(/\n/, $descr)) {
2498                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2499                 }
2500             } else {
2501                 $raw .= "#\n" if $pending;
2502             }
2503         }
2504
2505         foreach my $key (sort keys %$conf) {
2506             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2507             $raw .= "$key: $conf->{$key}\n";
2508         }
2509         return $raw;
2510     };
2511
2512     my $raw = &$generate_raw_config($conf);
2513
2514     if (scalar(keys %{$conf->{pending}})){
2515         $raw .= "\n[PENDING]\n";
2516         $raw .= &$generate_raw_config($conf->{pending}, 1);
2517     }
2518
2519     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2520         $raw .= "\n[special:cloudinit]\n";
2521         $raw .= &$generate_raw_config($conf->{cloudinit});
2522     }
2523
2524     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2525         $raw .= "\n[$snapname]\n";
2526         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2527     }
2528
2529     return $raw;
2530 }
2531
2532 sub load_defaults {
2533
2534     my $res = {};
2535
2536     # we use static defaults from our JSON schema configuration
2537     foreach my $key (keys %$confdesc) {
2538         if (defined(my $default = $confdesc->{$key}->{default})) {
2539             $res->{$key} = $default;
2540         }
2541     }
2542
2543     return $res;
2544 }
2545
2546 sub config_list {
2547     my $vmlist = PVE::Cluster::get_vmlist();
2548     my $res = {};
2549     return $res if !$vmlist || !$vmlist->{ids};
2550     my $ids = $vmlist->{ids};
2551     my $nodename = nodename();
2552
2553     foreach my $vmid (keys %$ids) {
2554         my $d = $ids->{$vmid};
2555         next if !$d->{node} || $d->{node} ne $nodename;
2556         next if !$d->{type} || $d->{type} ne 'qemu';
2557         $res->{$vmid}->{exists} = 1;
2558     }
2559     return $res;
2560 }
2561
2562 # test if VM uses local resources (to prevent migration)
2563 sub check_local_resources {
2564     my ($conf, $noerr) = @_;
2565
2566     my @loc_res = ();
2567     my $mapped_res = [];
2568
2569     my $nodelist = PVE::Cluster::get_nodelist();
2570     my $pci_map = PVE::Mapping::PCI::config();
2571     my $usb_map = PVE::Mapping::USB::config();
2572
2573     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2574
2575     my $add_missing_mapping = sub {
2576         my ($type, $key, $id) = @_;
2577         for my $node (@$nodelist) {
2578             my $entry;
2579             if ($type eq 'pci') {
2580                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2581             } elsif ($type eq 'usb') {
2582                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2583             }
2584             if (!scalar($entry->@*)) {
2585                 push @{$missing_mappings_by_node->{$node}}, $key;
2586             }
2587         }
2588     };
2589
2590     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2591     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2592
2593     push @loc_res, "ivshmem" if $conf->{ivshmem};
2594
2595     foreach my $k (keys %$conf) {
2596         if ($k =~ m/^usb/) {
2597             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2598             next if $entry->{host} =~ m/^spice$/i;
2599             if ($entry->{mapping}) {
2600                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2601                 push @$mapped_res, $k;
2602             }
2603         }
2604         if ($k =~ m/^hostpci/) {
2605             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2606             if ($entry->{mapping}) {
2607                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2608                 push @$mapped_res, $k;
2609             }
2610         }
2611         # sockets are safe: they will recreated be on the target side post-migrate
2612         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2613         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2614     }
2615
2616     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2617
2618     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2619 }
2620
2621 # check if used storages are available on all nodes (use by migrate)
2622 sub check_storage_availability {
2623     my ($storecfg, $conf, $node) = @_;
2624
2625     PVE::QemuConfig->foreach_volume($conf, sub {
2626         my ($ds, $drive) = @_;
2627
2628         my $volid = $drive->{file};
2629         return if !$volid;
2630
2631         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2632         return if !$sid;
2633
2634         # check if storage is available on both nodes
2635         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2636         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2637
2638         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2639
2640         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2641             if !$scfg->{content}->{$vtype};
2642    });
2643 }
2644
2645 # list nodes where all VM images are available (used by has_feature API)
2646 sub shared_nodes {
2647     my ($conf, $storecfg) = @_;
2648
2649     my $nodelist = PVE::Cluster::get_nodelist();
2650     my $nodehash = { map { $_ => 1 } @$nodelist };
2651     my $nodename = nodename();
2652
2653     PVE::QemuConfig->foreach_volume($conf, sub {
2654         my ($ds, $drive) = @_;
2655
2656         my $volid = $drive->{file};
2657         return if !$volid;
2658
2659         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2660         if ($storeid) {
2661             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2662             if ($scfg->{disable}) {
2663                 $nodehash = {};
2664             } elsif (my $avail = $scfg->{nodes}) {
2665                 foreach my $node (keys %$nodehash) {
2666                     delete $nodehash->{$node} if !$avail->{$node};
2667                 }
2668             } elsif (!$scfg->{shared}) {
2669                 foreach my $node (keys %$nodehash) {
2670                     delete $nodehash->{$node} if $node ne $nodename
2671                 }
2672             }
2673         }
2674     });
2675
2676     return $nodehash
2677 }
2678
2679 sub check_local_storage_availability {
2680     my ($conf, $storecfg) = @_;
2681
2682     my $nodelist = PVE::Cluster::get_nodelist();
2683     my $nodehash = { map { $_ => {} } @$nodelist };
2684
2685     PVE::QemuConfig->foreach_volume($conf, sub {
2686         my ($ds, $drive) = @_;
2687
2688         my $volid = $drive->{file};
2689         return if !$volid;
2690
2691         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2692         if ($storeid) {
2693             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2694
2695             if ($scfg->{disable}) {
2696                 foreach my $node (keys %$nodehash) {
2697                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2698                 }
2699             } elsif (my $avail = $scfg->{nodes}) {
2700                 foreach my $node (keys %$nodehash) {
2701                     if (!$avail->{$node}) {
2702                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2703                     }
2704                 }
2705             }
2706         }
2707     });
2708
2709     foreach my $node (values %$nodehash) {
2710         if (my $unavail = $node->{unavailable_storages}) {
2711             $node->{unavailable_storages} = [ sort keys %$unavail ];
2712         }
2713     }
2714
2715     return $nodehash
2716 }
2717
2718 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2719 sub check_running {
2720     my ($vmid, $nocheck, $node) = @_;
2721
2722     # $nocheck is set when called during a migration, in which case the config
2723     # file might still or already reside on the *other* node
2724     # - because rename has already happened, and current node is source
2725     # - because rename hasn't happened yet, and current node is target
2726     # - because rename has happened, current node is target, but hasn't yet
2727     # processed it yet
2728     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2729     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2730 }
2731
2732 sub vzlist {
2733
2734     my $vzlist = config_list();
2735
2736     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2737
2738     while (defined(my $de = $fd->read)) {
2739         next if $de !~ m/^(\d+)\.pid$/;
2740         my $vmid = $1;
2741         next if !defined($vzlist->{$vmid});
2742         if (my $pid = check_running($vmid)) {
2743             $vzlist->{$vmid}->{pid} = $pid;
2744         }
2745     }
2746
2747     return $vzlist;
2748 }
2749
2750 our $vmstatus_return_properties = {
2751     vmid => get_standard_option('pve-vmid'),
2752     status => {
2753         description => "QEMU process status.",
2754         type => 'string',
2755         enum => ['stopped', 'running'],
2756     },
2757     maxmem => {
2758         description => "Maximum memory in bytes.",
2759         type => 'integer',
2760         optional => 1,
2761         renderer => 'bytes',
2762     },
2763     maxdisk => {
2764         description => "Root disk size in bytes.",
2765         type => 'integer',
2766         optional => 1,
2767         renderer => 'bytes',
2768     },
2769     name => {
2770         description => "VM name.",
2771         type => 'string',
2772         optional => 1,
2773     },
2774     qmpstatus => {
2775         description => "VM run state from the 'query-status' QMP monitor command.",
2776         type => 'string',
2777         optional => 1,
2778     },
2779     pid => {
2780         description => "PID of running qemu process.",
2781         type => 'integer',
2782         optional => 1,
2783     },
2784     uptime => {
2785         description => "Uptime.",
2786         type => 'integer',
2787         optional => 1,
2788         renderer => 'duration',
2789     },
2790     cpus => {
2791         description => "Maximum usable CPUs.",
2792         type => 'number',
2793         optional => 1,
2794     },
2795     lock => {
2796         description => "The current config lock, if any.",
2797         type => 'string',
2798         optional => 1,
2799     },
2800     tags => {
2801         description  => "The current configured tags, if any",
2802         type => 'string',
2803         optional => 1,
2804     },
2805     'running-machine' => {
2806         description => "The currently running machine type (if running).",
2807         type => 'string',
2808         optional => 1,
2809     },
2810     'running-qemu' => {
2811         description => "The currently running QEMU version (if running).",
2812         type => 'string',
2813         optional => 1,
2814     },
2815 };
2816
2817 my $last_proc_pid_stat;
2818
2819 # get VM status information
2820 # This must be fast and should not block ($full == false)
2821 # We only query KVM using QMP if $full == true (this can be slow)
2822 sub vmstatus {
2823     my ($opt_vmid, $full) = @_;
2824
2825     my $res = {};
2826
2827     my $storecfg = PVE::Storage::config();
2828
2829     my $list = vzlist();
2830     my $defaults = load_defaults();
2831
2832     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2833
2834     my $cpucount = $cpuinfo->{cpus} || 1;
2835
2836     foreach my $vmid (keys %$list) {
2837         next if $opt_vmid && ($vmid ne $opt_vmid);
2838
2839         my $conf = PVE::QemuConfig->load_config($vmid);
2840
2841         my $d = { vmid => int($vmid) };
2842         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2843
2844         # fixme: better status?
2845         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2846
2847         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2848         if (defined($size)) {
2849             $d->{disk} = 0; # no info available
2850             $d->{maxdisk} = $size;
2851         } else {
2852             $d->{disk} = 0;
2853             $d->{maxdisk} = 0;
2854         }
2855
2856         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2857             * ($conf->{cores} || $defaults->{cores});
2858         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2859         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2860
2861         $d->{name} = $conf->{name} || "VM $vmid";
2862         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2863
2864         if ($conf->{balloon}) {
2865             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2866             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2867                 : $defaults->{shares};
2868         }
2869
2870         $d->{uptime} = 0;
2871         $d->{cpu} = 0;
2872         $d->{mem} = 0;
2873
2874         $d->{netout} = 0;
2875         $d->{netin} = 0;
2876
2877         $d->{diskread} = 0;
2878         $d->{diskwrite} = 0;
2879
2880         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2881
2882         $d->{serial} = 1 if conf_has_serial($conf);
2883         $d->{lock} = $conf->{lock} if $conf->{lock};
2884         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2885
2886         $res->{$vmid} = $d;
2887     }
2888
2889     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2890     foreach my $dev (keys %$netdev) {
2891         next if $dev !~ m/^tap([1-9]\d*)i/;
2892         my $vmid = $1;
2893         my $d = $res->{$vmid};
2894         next if !$d;
2895
2896         $d->{netout} += $netdev->{$dev}->{receive};
2897         $d->{netin} += $netdev->{$dev}->{transmit};
2898
2899         if ($full) {
2900             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2901             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2902         }
2903
2904     }
2905
2906     my $ctime = gettimeofday;
2907
2908     foreach my $vmid (keys %$list) {
2909
2910         my $d = $res->{$vmid};
2911         my $pid = $d->{pid};
2912         next if !$pid;
2913
2914         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2915         next if !$pstat; # not running
2916
2917         my $used = $pstat->{utime} + $pstat->{stime};
2918
2919         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2920
2921         if ($pstat->{vsize}) {
2922             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2923         }
2924
2925         my $old = $last_proc_pid_stat->{$pid};
2926         if (!$old) {
2927             $last_proc_pid_stat->{$pid} = {
2928                 time => $ctime,
2929                 used => $used,
2930                 cpu => 0,
2931             };
2932             next;
2933         }
2934
2935         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2936
2937         if ($dtime > 1000) {
2938             my $dutime = $used -  $old->{used};
2939
2940             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2941             $last_proc_pid_stat->{$pid} = {
2942                 time => $ctime,
2943                 used => $used,
2944                 cpu => $d->{cpu},
2945             };
2946         } else {
2947             $d->{cpu} = $old->{cpu};
2948         }
2949     }
2950
2951     return $res if !$full;
2952
2953     my $qmpclient = PVE::QMPClient->new();
2954
2955     my $ballooncb = sub {
2956         my ($vmid, $resp) = @_;
2957
2958         my $info = $resp->{'return'};
2959         return if !$info->{max_mem};
2960
2961         my $d = $res->{$vmid};
2962
2963         # use memory assigned to VM
2964         $d->{maxmem} = $info->{max_mem};
2965         $d->{balloon} = $info->{actual};
2966
2967         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2968             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2969             $d->{freemem} = $info->{free_mem};
2970         }
2971
2972         $d->{ballooninfo} = $info;
2973     };
2974
2975     my $blockstatscb = sub {
2976         my ($vmid, $resp) = @_;
2977         my $data = $resp->{'return'} || [];
2978         my $totalrdbytes = 0;
2979         my $totalwrbytes = 0;
2980
2981         for my $blockstat (@$data) {
2982             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2983             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2984
2985             $blockstat->{device} =~ s/drive-//;
2986             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2987         }
2988         $res->{$vmid}->{diskread} = $totalrdbytes;
2989         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2990     };
2991
2992     my $machinecb = sub {
2993         my ($vmid, $resp) = @_;
2994         my $data = $resp->{'return'} || [];
2995
2996         $res->{$vmid}->{'running-machine'} =
2997             PVE::QemuServer::Machine::current_from_query_machines($data);
2998     };
2999
3000     my $versioncb = sub {
3001         my ($vmid, $resp) = @_;
3002         my $data = $resp->{'return'} // {};
3003         my $version = 'unknown';
3004
3005         if (my $v = $data->{qemu}) {
3006             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3007         }
3008
3009         $res->{$vmid}->{'running-qemu'} = $version;
3010     };
3011
3012     my $statuscb = sub {
3013         my ($vmid, $resp) = @_;
3014
3015         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3016         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3017         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3018         # this fails if ballon driver is not loaded, so this must be
3019         # the last commnand (following command are aborted if this fails).
3020         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3021
3022         my $status = 'unknown';
3023         if (!defined($status = $resp->{'return'}->{status})) {
3024             warn "unable to get VM status\n";
3025             return;
3026         }
3027
3028         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3029     };
3030
3031     foreach my $vmid (keys %$list) {
3032         next if $opt_vmid && ($vmid ne $opt_vmid);
3033         next if !$res->{$vmid}->{pid}; # not running
3034         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3035     }
3036
3037     $qmpclient->queue_execute(undef, 2);
3038
3039     foreach my $vmid (keys %$list) {
3040         next if $opt_vmid && ($vmid ne $opt_vmid);
3041         next if !$res->{$vmid}->{pid}; #not running
3042
3043         # we can't use the $qmpclient since it might have already aborted on
3044         # 'query-balloon', but this might also fail for older versions...
3045         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3046         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3047     }
3048
3049     foreach my $vmid (keys %$list) {
3050         next if $opt_vmid && ($vmid ne $opt_vmid);
3051         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3052     }
3053
3054     return $res;
3055 }
3056
3057 sub conf_has_serial {
3058     my ($conf) = @_;
3059
3060     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3061         if ($conf->{"serial$i"}) {
3062             return 1;
3063         }
3064     }
3065
3066     return 0;
3067 }
3068
3069 sub conf_has_audio {
3070     my ($conf, $id) = @_;
3071
3072     $id //= 0;
3073     my $audio = $conf->{"audio$id"};
3074     return if !defined($audio);
3075
3076     my $audioproperties = parse_property_string($audio_fmt, $audio);
3077     my $audiodriver = $audioproperties->{driver} // 'spice';
3078
3079     return {
3080         dev => $audioproperties->{device},
3081         dev_id => "audiodev$id",
3082         backend => $audiodriver,
3083         backend_id => "$audiodriver-backend${id}",
3084     };
3085 }
3086
3087 sub audio_devs {
3088     my ($audio, $audiopciaddr, $machine_version) = @_;
3089
3090     my $devs = [];
3091
3092     my $id = $audio->{dev_id};
3093     my $audiodev = "";
3094     if (min_version($machine_version, 4, 2)) {
3095         $audiodev = ",audiodev=$audio->{backend_id}";
3096     }
3097
3098     if ($audio->{dev} eq 'AC97') {
3099         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3100     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3101         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3102         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3103         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3104     } else {
3105         die "unkown audio device '$audio->{dev}', implement me!";
3106     }
3107
3108     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3109
3110     return $devs;
3111 }
3112
3113 sub get_tpm_paths {
3114     my ($vmid) = @_;
3115     return {
3116         socket => "/var/run/qemu-server/$vmid.swtpm",
3117         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3118     };
3119 }
3120
3121 sub add_tpm_device {
3122     my ($vmid, $devices, $conf) = @_;
3123
3124     return if !$conf->{tpmstate0};
3125
3126     my $paths = get_tpm_paths($vmid);
3127
3128     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3129     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3130     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3131 }
3132
3133 sub start_swtpm {
3134     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3135
3136     return if !$tpmdrive;
3137
3138     my $state;
3139     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3140     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3141     if ($storeid) {
3142         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3143     } else {
3144         $state = $tpm->{file};
3145     }
3146
3147     my $paths = get_tpm_paths($vmid);
3148
3149     # during migration, we will get state from remote
3150     #
3151     if (!$migration) {
3152         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3153         my $setup_cmd = [
3154             "swtpm_setup",
3155             "--tpmstate",
3156             "file://$state",
3157             "--createek",
3158             "--create-ek-cert",
3159             "--create-platform-cert",
3160             "--lock-nvram",
3161             "--config",
3162             "/etc/swtpm_setup.conf", # do not use XDG configs
3163             "--runas",
3164             "0", # force creation as root, error if not possible
3165             "--not-overwrite", # ignore existing state, do not modify
3166         ];
3167
3168         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3169         # TPM 2.0 supports ECC crypto, use if possible
3170         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3171
3172         run_command($setup_cmd, outfunc => sub {
3173             print "swtpm_setup: $1\n";
3174         });
3175     }
3176
3177     # Used to distinguish different invocations in the log.
3178     my $log_prefix = "[id=" . int(time()) . "] ";
3179
3180     my $emulator_cmd = [
3181         "swtpm",
3182         "socket",
3183         "--tpmstate",
3184         "backend-uri=file://$state,mode=0600",
3185         "--ctrl",
3186         "type=unixio,path=$paths->{socket},mode=0600",
3187         "--pid",
3188         "file=$paths->{pid}",
3189         "--terminate", # terminate on QEMU disconnect
3190         "--daemon",
3191         "--log",
3192         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3193     ];
3194     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3195     run_command($emulator_cmd, outfunc => sub { print $1; });
3196
3197     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3198     while (! -e $paths->{pid}) {
3199         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3200         usleep(50_000);
3201     }
3202
3203     # return untainted PID of swtpm daemon so it can be killed on error
3204     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3205     return $1;
3206 }
3207
3208 sub vga_conf_has_spice {
3209     my ($vga) = @_;
3210
3211     my $vgaconf = parse_vga($vga);
3212     my $vgatype = $vgaconf->{type};
3213     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3214
3215     return $1 || 1;
3216 }
3217
3218 sub is_native($) {
3219     my ($arch) = @_;
3220     return get_host_arch() eq $arch;
3221 }
3222
3223 sub get_vm_arch {
3224     my ($conf) = @_;
3225     return $conf->{arch} // get_host_arch();
3226 }
3227
3228 my $default_machines = {
3229     x86_64 => 'pc',
3230     aarch64 => 'virt',
3231 };
3232
3233 sub get_installed_machine_version {
3234     my ($kvmversion) = @_;
3235     $kvmversion = kvm_user_version() if !defined($kvmversion);
3236     $kvmversion =~ m/^(\d+\.\d+)/;
3237     return $1;
3238 }
3239
3240 sub windows_get_pinned_machine_version {
3241     my ($machine, $base_version, $kvmversion) = @_;
3242
3243     my $pin_version = $base_version;
3244     if (!defined($base_version) ||
3245         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3246     ) {
3247         $pin_version = get_installed_machine_version($kvmversion);
3248     }
3249     if (!$machine || $machine eq 'pc') {
3250         $machine = "pc-i440fx-$pin_version";
3251     } elsif ($machine eq 'q35') {
3252         $machine = "pc-q35-$pin_version";
3253     } elsif ($machine eq 'virt') {
3254         $machine = "virt-$pin_version";
3255     } else {
3256         warn "unknown machine type '$machine', not touching that!\n";
3257     }
3258
3259     return $machine;
3260 }
3261
3262 sub get_vm_machine {
3263     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3264
3265     my $machine = $forcemachine || $conf->{machine};
3266
3267     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3268         $kvmversion //= kvm_user_version();
3269         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3270         # layout which confuses windows quite a bit and may result in various regressions..
3271         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3272         if (windows_version($conf->{ostype})) {
3273             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3274         }
3275         $arch //= 'x86_64';
3276         $machine ||= $default_machines->{$arch};
3277         if ($add_pve_version) {
3278             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3279             $machine .= "+pve$pvever";
3280         }
3281     }
3282
3283     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3284         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3285         $machine = $1 if $is_pxe;
3286
3287         # for version-pinned machines that do not include a pve-version (e.g.
3288         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3289         $machine .= '+pve0';
3290
3291         $machine .= '.pxe' if $is_pxe;
3292     }
3293
3294     return $machine;
3295 }
3296
3297 sub get_ovmf_files($$$) {
3298     my ($arch, $efidisk, $smm) = @_;
3299
3300     my $types = $OVMF->{$arch}
3301         or die "no OVMF images known for architecture '$arch'\n";
3302
3303     my $type = 'default';
3304     if ($arch eq 'x86_64') {
3305         if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3306             $type = $smm ? "4m" : "4m-no-smm";
3307             $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3308         } else {
3309             # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3310         }
3311     }
3312
3313     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3314     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3315     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3316
3317     return ($ovmf_code, $ovmf_vars);
3318 }
3319
3320 my $Arch2Qemu = {
3321     aarch64 => '/usr/bin/qemu-system-aarch64',
3322     x86_64 => '/usr/bin/qemu-system-x86_64',
3323 };
3324 sub get_command_for_arch($) {
3325     my ($arch) = @_;
3326     return '/usr/bin/kvm' if is_native($arch);
3327
3328     my $cmd = $Arch2Qemu->{$arch}
3329         or die "don't know how to emulate architecture '$arch'\n";
3330     return $cmd;
3331 }
3332
3333 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3334 # to use in a QEMU command line (-cpu element), first array_intersect the result
3335 # of query_supported_ with query_understood_. This is necessary because:
3336 #
3337 # a) query_understood_ returns flags the host cannot use and
3338 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3339 #    flags, but CPU settings - with most of them being flags. Those settings
3340 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3341 #
3342 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3343 # expensive. If you need the value returned from this, you can get it much
3344 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3345 # $accel being 'kvm' or 'tcg'.
3346 #
3347 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3348 # changes, automatically populating pmxcfs.
3349 #
3350 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3351 # since kvm and tcg machines support different flags
3352 #
3353 sub query_supported_cpu_flags {
3354     my ($arch) = @_;
3355
3356     $arch //= get_host_arch();
3357     my $default_machine = $default_machines->{$arch};
3358
3359     my $flags = {};
3360
3361     # FIXME: Once this is merged, the code below should work for ARM as well:
3362     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3363     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3364         $arch eq "aarch64";
3365
3366     my $kvm_supported = defined(kvm_version());
3367     my $qemu_cmd = get_command_for_arch($arch);
3368     my $fakevmid = -1;
3369     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3370
3371     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3372     my $query_supported_run_qemu = sub {
3373         my ($kvm) = @_;
3374
3375         my $flags = {};
3376         my $cmd = [
3377             $qemu_cmd,
3378             '-machine', $default_machine,
3379             '-display', 'none',
3380             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3381             '-mon', 'chardev=qmp,mode=control',
3382             '-pidfile', $pidfile,
3383             '-S', '-daemonize'
3384         ];
3385
3386         if (!$kvm) {
3387             push @$cmd, '-accel', 'tcg';
3388         }
3389
3390         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3391         die "QEMU flag querying VM exited with code " . $rc if $rc;
3392
3393         eval {
3394             my $cmd_result = mon_cmd(
3395                 $fakevmid,
3396                 'query-cpu-model-expansion',
3397                 type => 'full',
3398                 model => { name => 'host' }
3399             );
3400
3401             my $props = $cmd_result->{model}->{props};
3402             foreach my $prop (keys %$props) {
3403                 next if $props->{$prop} ne '1';
3404                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3405                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3406                 # We only keep those with underscores, to match /proc/cpuinfo
3407                 $prop =~ s/\.|-/_/g;
3408                 $flags->{$prop} = 1;
3409             }
3410         };
3411         my $err = $@;
3412
3413         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3414         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3415
3416         die $err if $err;
3417
3418         return [ sort keys %$flags ];
3419     };
3420
3421     # We need to query QEMU twice, since KVM and TCG have different supported flags
3422     PVE::QemuConfig->lock_config($fakevmid, sub {
3423         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3424         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3425
3426         if ($kvm_supported) {
3427             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3428             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3429         }
3430     });
3431
3432     return $flags;
3433 }
3434
3435 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3436 my $understood_cpu_flag_dir = "/usr/share/kvm";
3437 sub query_understood_cpu_flags {
3438     my $arch = get_host_arch();
3439     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3440
3441     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3442         if ! -e $filepath;
3443
3444     my $raw = file_get_contents($filepath);
3445     $raw =~ s/^\s+|\s+$//g;
3446     my @flags = split(/\s+/, $raw);
3447
3448     return \@flags;
3449 }
3450
3451 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3452 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3453 my sub should_disable_smm {
3454     my ($conf, $vga, $machine) = @_;
3455
3456     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3457
3458     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3459         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3460 }
3461
3462 my sub print_ovmf_drive_commandlines {
3463     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3464
3465     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3466
3467     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3468
3469     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3470     if ($d) {
3471         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3472         my ($path, $format) = $d->@{'file', 'format'};
3473         if ($storeid) {
3474             $path = PVE::Storage::path($storecfg, $d->{file});
3475             if (!defined($format)) {
3476                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3477                 $format = qemu_img_format($scfg, $volname);
3478             }
3479         } elsif (!defined($format)) {
3480             die "efidisk format must be specified\n";
3481         }
3482         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3483         if ($path =~ m/^rbd:/) {
3484             $var_drive_str .= ',cache=writeback';
3485             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3486         }
3487         $var_drive_str .= ",format=$format,file=$path";
3488
3489         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3490         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3491     } else {
3492         log_warn("no efidisk configured! Using temporary efivars disk.");
3493         my $path = "/tmp/$vmid-ovmf.fd";
3494         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3495         $var_drive_str .= ",format=raw,file=$path";
3496         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3497     }
3498
3499     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3500 }
3501
3502 sub config_to_command {
3503     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3504         $pbs_backing) = @_;
3505
3506     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3507     my $devices = [];
3508     my $bridges = {};
3509     my $ostype = $conf->{ostype};
3510     my $winversion = windows_version($ostype);
3511     my $kvm = $conf->{kvm};
3512     my $nodename = nodename();
3513
3514     my $arch = get_vm_arch($conf);
3515     my $kvm_binary = get_command_for_arch($arch);
3516     my $kvmver = kvm_user_version($kvm_binary);
3517
3518     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3519         $kvmver //= "undefined";
3520         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3521     }
3522
3523     my $add_pve_version = min_version($kvmver, 4, 1);
3524
3525     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3526     my $machine_version = extract_version($machine_type, $kvmver);
3527     $kvm //= 1 if is_native($arch);
3528
3529     $machine_version =~ m/(\d+)\.(\d+)/;
3530     my ($machine_major, $machine_minor) = ($1, $2);
3531
3532     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3533         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3534     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3535         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3536             ." please upgrade node '$nodename'\n"
3537     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3538         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3539         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3540             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3541             ." node '$nodename'\n";
3542     }
3543
3544     # if a specific +pve version is required for a feature, use $version_guard
3545     # instead of min_version to allow machines to be run with the minimum
3546     # required version
3547     my $required_pve_version = 0;
3548     my $version_guard = sub {
3549         my ($major, $minor, $pve) = @_;
3550         return 0 if !min_version($machine_version, $major, $minor, $pve);
3551         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3552         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3553         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3554         return 1;
3555     };
3556
3557     if ($kvm && !defined kvm_version()) {
3558         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3559             ." or enable in BIOS.\n";
3560     }
3561
3562     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3563     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3564     my $use_old_bios_files = undef;
3565     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3566
3567     my $cmd = [];
3568     if ($conf->{affinity}) {
3569         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3570     }
3571
3572     push @$cmd, $kvm_binary;
3573
3574     push @$cmd, '-id', $vmid;
3575
3576     my $vmname = $conf->{name} || "vm$vmid";
3577
3578     push @$cmd, '-name', "$vmname,debug-threads=on";
3579
3580     push @$cmd, '-no-shutdown';
3581
3582     my $use_virtio = 0;
3583
3584     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3585     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3586     push @$cmd, '-mon', "chardev=qmp,mode=control";
3587
3588     if (min_version($machine_version, 2, 12)) {
3589         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3590         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3591     }
3592
3593     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3594
3595     push @$cmd, '-daemonize';
3596
3597     if ($conf->{smbios1}) {
3598         my $smbios_conf = parse_smbios1($conf->{smbios1});
3599         if ($smbios_conf->{base64}) {
3600             # Do not pass base64 flag to qemu
3601             delete $smbios_conf->{base64};
3602             my $smbios_string = "";
3603             foreach my $key (keys %$smbios_conf) {
3604                 my $value;
3605                 if ($key eq "uuid") {
3606                     $value = $smbios_conf->{uuid}
3607                 } else {
3608                     $value = decode_base64($smbios_conf->{$key});
3609                 }
3610                 # qemu accepts any binary data, only commas need escaping by double comma
3611                 $value =~ s/,/,,/g;
3612                 $smbios_string .= "," . $key . "=" . $value if $value;
3613             }
3614             push @$cmd, '-smbios', "type=1" . $smbios_string;
3615         } else {
3616             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3617         }
3618     }
3619
3620     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3621         my ($code_drive_str, $var_drive_str) =
3622             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3623         push $cmd->@*, '-drive', $code_drive_str;
3624         push $cmd->@*, '-drive', $var_drive_str;
3625     }
3626
3627     if ($q35) { # tell QEMU to load q35 config early
3628         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3629         if (min_version($machine_version, 4, 0)) {
3630             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3631         } else {
3632             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3633         }
3634     }
3635
3636     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3637         push @$cmd, $fixups->@*;
3638     }
3639
3640     if ($conf->{vmgenid}) {
3641         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3642     }
3643
3644     # add usb controllers
3645     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3646         $conf, $bridges, $arch, $machine_type, $machine_version);
3647     push @$devices, @usbcontrollers if @usbcontrollers;
3648     my $vga = parse_vga($conf->{vga});
3649
3650     my $qxlnum = vga_conf_has_spice($conf->{vga});
3651     $vga->{type} = 'qxl' if $qxlnum;
3652
3653     if (!$vga->{type}) {
3654         if ($arch eq 'aarch64') {
3655             $vga->{type} = 'virtio';
3656         } elsif (min_version($machine_version, 2, 9)) {
3657             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3658         } else {
3659             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3660         }
3661     }
3662
3663     # enable absolute mouse coordinates (needed by vnc)
3664     my $tablet = $conf->{tablet};
3665     if (!defined($tablet)) {
3666         $tablet = $defaults->{tablet};
3667         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3668         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3669     }
3670
3671     if ($tablet) {
3672         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3673         my $kbd = print_keyboarddevice_full($conf, $arch);
3674         push @$devices, '-device', $kbd if defined($kbd);
3675     }
3676
3677     my $bootorder = device_bootorder($conf);
3678
3679     # host pci device passthrough
3680     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3681         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3682
3683     # usb devices
3684     my $usb_dev_features = {};
3685     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3686
3687     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3688         $conf, $usb_dev_features, $bootorder, $machine_version);
3689     push @$devices, @usbdevices if @usbdevices;
3690
3691     # serial devices
3692     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3693         my $path = $conf->{"serial$i"} or next;
3694         if ($path eq 'socket') {
3695             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3696             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3697             # On aarch64, serial0 is the UART device. QEMU only allows
3698             # connecting UART devices via the '-serial' command line, as
3699             # the device has a fixed slot on the hardware...
3700             if ($arch eq 'aarch64' && $i == 0) {
3701                 push @$devices, '-serial', "chardev:serial$i";
3702             } else {
3703                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3704             }
3705         } else {
3706             die "no such serial device\n" if ! -c $path;
3707             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3708             push @$devices, '-device', "isa-serial,chardev=serial$i";
3709         }
3710     }
3711
3712     # parallel devices
3713     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3714         if (my $path = $conf->{"parallel$i"}) {
3715             die "no such parallel device\n" if ! -c $path;
3716             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3717             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3718             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3719         }
3720     }
3721
3722     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3723         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3724         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3725         push @$devices, @$audio_devs;
3726     }
3727
3728     # Add a TPM only if the VM is not a template,
3729     # to support backing up template VMs even if the TPM disk is write-protected.
3730     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3731
3732     my $sockets = 1;
3733     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3734     $sockets = $conf->{sockets} if  $conf->{sockets};
3735
3736     my $cores = $conf->{cores} || 1;
3737
3738     my $maxcpus = $sockets * $cores;
3739
3740     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3741
3742     my $allowed_vcpus = $cpuinfo->{cpus};
3743
3744     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3745
3746     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3747         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3748         for (my $i = 2; $i <= $vcpus; $i++)  {
3749             my $cpustr = print_cpu_device($conf,$i);
3750             push @$cmd, '-device', $cpustr;
3751         }
3752
3753     } else {
3754
3755         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3756     }
3757     push @$cmd, '-nodefaults';
3758
3759     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3760
3761     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3762
3763     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3764
3765     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3766         push @$devices, '-device', print_vga_device(
3767             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3768
3769         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3770
3771         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3772         push @$cmd,  '-vnc', "unix:$socket,password=on";
3773     } else {
3774         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3775         push @$cmd, '-nographic';
3776     }
3777
3778     # time drift fix
3779     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3780     my $useLocaltime = $conf->{localtime};
3781
3782     if ($winversion >= 5) { # windows
3783         $useLocaltime = 1 if !defined($conf->{localtime});
3784
3785         # use time drift fix when acpi is enabled
3786         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3787             $tdf = 1 if !defined($conf->{tdf});
3788         }
3789     }
3790
3791     if ($winversion >= 6) {
3792         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3793         push @$machineFlags, 'hpet=off';
3794     }
3795
3796     push @$rtcFlags, 'driftfix=slew' if $tdf;
3797
3798     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3799         push @$rtcFlags, "base=$conf->{startdate}";
3800     } elsif ($useLocaltime) {
3801         push @$rtcFlags, 'base=localtime';
3802     }
3803
3804     if ($forcecpu) {
3805         push @$cmd, '-cpu', $forcecpu;
3806     } else {
3807         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3808     }
3809
3810     PVE::QemuServer::Memory::config(
3811         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3812
3813     push @$cmd, '-S' if $conf->{freeze};
3814
3815     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3816
3817     my $guest_agent = parse_guest_agent($conf);
3818
3819     if ($guest_agent->{enabled}) {
3820         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3821         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3822
3823         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3824             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3825             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3826             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3827         } elsif ($guest_agent->{type} eq 'isa') {
3828             push @$devices, '-device', "isa-serial,chardev=qga0";
3829         }
3830     }
3831
3832     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3833     if ($rng && $version_guard->(4, 1, 2)) {
3834         check_rng_source($rng->{source});
3835
3836         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3837         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3838         my $limiter_str = "";
3839         if ($max_bytes) {
3840             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3841         }
3842
3843         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3844         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3845         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3846     }
3847
3848     my $spice_port;
3849
3850     assert_clipboard_config($vga);
3851     my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3852
3853     if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3854         if ($qxlnum > 1) {
3855             if ($winversion){
3856                 for (my $i = 1; $i < $qxlnum; $i++){
3857                     push @$devices, '-device', print_vga_device(
3858                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3859                 }
3860             } else {
3861                 # assume other OS works like Linux
3862                 my ($ram, $vram) = ("134217728", "67108864");
3863                 if ($vga->{memory}) {
3864                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3865                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3866                 }
3867                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3868                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3869             }
3870         }
3871
3872         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3873
3874         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3875         if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3876             push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3877         } else {
3878             push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3879         }
3880         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3881
3882         if ($is_spice) {
3883             my $pfamily = PVE::Tools::get_host_address_family($nodename);
3884             my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3885             die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3886
3887             my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3888             $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3889
3890             my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3891             my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3892             if ($spice_enhancement->{foldersharing}) {
3893                 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3894                 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3895             }
3896
3897             my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3898             $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3899                 if $spice_enhancement->{videostreaming};
3900             push @$devices, '-spice', "$spice_opts";
3901         }
3902     }
3903
3904     # enable balloon by default, unless explicitly disabled
3905     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3906         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3907         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3908         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3909         push @$devices, '-device', $ballooncmd;
3910     }
3911
3912     if ($conf->{watchdog}) {
3913         my $wdopts = parse_watchdog($conf->{watchdog});
3914         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3915         my $watchdog = $wdopts->{model} || 'i6300esb';
3916         push @$devices, '-device', "$watchdog$pciaddr";
3917         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3918     }
3919
3920     my $vollist = [];
3921     my $scsicontroller = {};
3922     my $ahcicontroller = {};
3923     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3924
3925     # Add iscsi initiator name if available
3926     if (my $initiator = get_initiator_name()) {
3927         push @$devices, '-iscsi', "initiator-name=$initiator";
3928     }
3929
3930     PVE::QemuConfig->foreach_volume($conf, sub {
3931         my ($ds, $drive) = @_;
3932
3933         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3934             check_volume_storage_type($storecfg, $drive->{file});
3935             push @$vollist, $drive->{file};
3936         }
3937
3938         # ignore efidisk here, already added in bios/fw handling code above
3939         return if $drive->{interface} eq 'efidisk';
3940         # similar for TPM
3941         return if $drive->{interface} eq 'tpmstate';
3942
3943         $use_virtio = 1 if $ds =~ m/^virtio/;
3944
3945         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3946
3947         if ($drive->{interface} eq 'virtio'){
3948            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3949         }
3950
3951         if ($drive->{interface} eq 'scsi') {
3952
3953             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3954
3955             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3956                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3957
3958             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3959             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3960
3961             my $iothread = '';
3962             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3963                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3964                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3965             } elsif ($drive->{iothread}) {
3966                 log_warn(
3967                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3968                 );
3969             }
3970
3971             my $queues = '';
3972             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3973                 $queues = ",num_queues=$drive->{queues}";
3974             }
3975
3976             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3977                 if !$scsicontroller->{$controller};
3978             $scsicontroller->{$controller}=1;
3979         }
3980
3981         if ($drive->{interface} eq 'sata') {
3982             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3983             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3984             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3985                 if !$ahcicontroller->{$controller};
3986             $ahcicontroller->{$controller}=1;
3987         }
3988
3989         my $pbs_conf = $pbs_backing->{$ds};
3990         my $pbs_name = undef;
3991         if ($pbs_conf) {
3992             $pbs_name = "drive-$ds-pbs";
3993             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3994         }
3995
3996         my $drive_cmd = print_drive_commandline_full(
3997             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3998
3999         # extra protection for templates, but SATA and IDE don't support it..
4000         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4001
4002         push @$devices, '-drive',$drive_cmd;
4003         push @$devices, '-device', print_drivedevice_full(
4004             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4005     });
4006
4007     for (my $i = 0; $i < $MAX_NETS; $i++) {
4008         my $netname = "net$i";
4009
4010         next if !$conf->{$netname};
4011         my $d = parse_net($conf->{$netname});
4012         next if !$d;
4013         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4014
4015         $use_virtio = 1 if $d->{model} eq 'virtio';
4016
4017         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4018
4019         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4020         push @$devices, '-netdev', $netdevfull;
4021
4022         my $netdevicefull = print_netdevice_full(
4023             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4024
4025         push @$devices, '-device', $netdevicefull;
4026     }
4027
4028     if ($conf->{ivshmem}) {
4029         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4030
4031         my $bus;
4032         if ($q35) {
4033             $bus = print_pcie_addr("ivshmem");
4034         } else {
4035             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4036         }
4037
4038         my $ivshmem_name = $ivshmem->{name} // $vmid;
4039         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4040
4041         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4042         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4043             .",size=$ivshmem->{size}M";
4044     }
4045
4046     # pci.4 is nested in pci.1
4047     $bridges->{1} = 1 if $bridges->{4};
4048
4049     if (!$q35) { # add pci bridges
4050         if (min_version($machine_version, 2, 3)) {
4051            $bridges->{1} = 1;
4052            $bridges->{2} = 1;
4053         }
4054         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4055     }
4056
4057     for my $k (sort {$b cmp $a} keys %$bridges) {
4058         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4059
4060         my $k_name = $k;
4061         if ($k == 2 && $legacy_igd) {
4062             $k_name = "$k-igd";
4063         }
4064         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4065         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4066
4067         if ($q35) { # add after -readconfig pve-q35.cfg
4068             splice @$devices, 2, 0, '-device', $devstr;
4069         } else {
4070             unshift @$devices, '-device', $devstr if $k > 0;
4071         }
4072     }
4073
4074     if (!$kvm) {
4075         push @$machineFlags, 'accel=tcg';
4076     }
4077
4078     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4079
4080     my $machine_type_min = $machine_type;
4081     if ($add_pve_version) {
4082         $machine_type_min =~ s/\+pve\d+$//;
4083         $machine_type_min .= "+pve$required_pve_version";
4084     }
4085     push @$machineFlags, "type=${machine_type_min}";
4086
4087     push @$cmd, @$devices;
4088     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4089     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4090     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4091
4092     if (my $vmstate = $conf->{vmstate}) {
4093         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4094         push @$vollist, $vmstate;
4095         push @$cmd, '-loadstate', $statepath;
4096         print "activating and using '$vmstate' as vmstate\n";
4097     }
4098
4099     if (PVE::QemuConfig->is_template($conf)) {
4100         # needed to workaround base volumes being read-only
4101         push @$cmd, '-snapshot';
4102     }
4103
4104     # add custom args
4105     if ($conf->{args}) {
4106         my $aa = PVE::Tools::split_args($conf->{args});
4107         push @$cmd, @$aa;
4108     }
4109
4110     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4111 }
4112
4113 sub check_rng_source {
4114     my ($source) = @_;
4115
4116     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4117     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4118         if ! -e $source;
4119
4120     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4121     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4122         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4123         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4124         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4125             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4126             ." to the host.\n";
4127     }
4128 }
4129
4130 sub spice_port {
4131     my ($vmid) = @_;
4132
4133     my $res = mon_cmd($vmid, 'query-spice');
4134
4135     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4136 }
4137
4138 sub vm_devices_list {
4139     my ($vmid) = @_;
4140
4141     my $res = mon_cmd($vmid, 'query-pci');
4142     my $devices_to_check = [];
4143     my $devices = {};
4144     foreach my $pcibus (@$res) {
4145         push @$devices_to_check, @{$pcibus->{devices}},
4146     }
4147
4148     while (@$devices_to_check) {
4149         my $to_check = [];
4150         for my $d (@$devices_to_check) {
4151             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4152             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4153
4154             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4155             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4156         }
4157         $devices_to_check = $to_check;
4158     }
4159
4160     my $resblock = mon_cmd($vmid, 'query-block');
4161     foreach my $block (@$resblock) {
4162         if($block->{device} =~ m/^drive-(\S+)/){
4163                 $devices->{$1} = 1;
4164         }
4165     }
4166
4167     my $resmice = mon_cmd($vmid, 'query-mice');
4168     foreach my $mice (@$resmice) {
4169         if ($mice->{name} eq 'QEMU HID Tablet') {
4170             $devices->{tablet} = 1;
4171             last;
4172         }
4173     }
4174
4175     # for usb devices there is no query-usb
4176     # but we can iterate over the entries in
4177     # qom-list path=/machine/peripheral
4178     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4179     foreach my $per (@$resperipheral) {
4180         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4181             $devices->{$per->{name}} = 1;
4182         }
4183     }
4184
4185     return $devices;
4186 }
4187
4188 sub vm_deviceplug {
4189     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4190
4191     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4192
4193     my $devices_list = vm_devices_list($vmid);
4194     return 1 if defined($devices_list->{$deviceid});
4195
4196     # add PCI bridge if we need it for the device
4197     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4198
4199     if ($deviceid eq 'tablet') {
4200         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4201     } elsif ($deviceid eq 'keyboard') {
4202         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4203     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4204         my $id = $1;
4205         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4206         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4207     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4208         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4209     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4210         qemu_iothread_add($vmid, $deviceid, $device);
4211
4212         qemu_driveadd($storecfg, $vmid, $device);
4213         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4214
4215         qemu_deviceadd($vmid, $devicefull);
4216         eval { qemu_deviceaddverify($vmid, $deviceid); };
4217         if (my $err = $@) {
4218             eval { qemu_drivedel($vmid, $deviceid); };
4219             warn $@ if $@;
4220             die $err;
4221         }
4222     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4223         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4224         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4225         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4226
4227         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4228
4229         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4230             qemu_iothread_add($vmid, $deviceid, $device);
4231             $devicefull .= ",iothread=iothread-$deviceid";
4232         }
4233
4234         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4235             $devicefull .= ",num_queues=$device->{queues}";
4236         }
4237
4238         qemu_deviceadd($vmid, $devicefull);
4239         qemu_deviceaddverify($vmid, $deviceid);
4240     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4241         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4242         qemu_driveadd($storecfg, $vmid, $device);
4243
4244         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4245         eval { qemu_deviceadd($vmid, $devicefull); };
4246         if (my $err = $@) {
4247             eval { qemu_drivedel($vmid, $deviceid); };
4248             warn $@ if $@;
4249             die $err;
4250         }
4251     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4252         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4253
4254         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4255         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4256         my $use_old_bios_files = undef;
4257         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4258
4259         my $netdevicefull = print_netdevice_full(
4260             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4261         qemu_deviceadd($vmid, $netdevicefull);
4262         eval {
4263             qemu_deviceaddverify($vmid, $deviceid);
4264             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4265         };
4266         if (my $err = $@) {
4267             eval { qemu_netdevdel($vmid, $deviceid); };
4268             warn $@ if $@;
4269             die $err;
4270         }
4271     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4272         my $bridgeid = $2;
4273         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4274         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4275
4276         qemu_deviceadd($vmid, $devicefull);
4277         qemu_deviceaddverify($vmid, $deviceid);
4278     } else {
4279         die "can't hotplug device '$deviceid'\n";
4280     }
4281
4282     return 1;
4283 }
4284
4285 # fixme: this should raise exceptions on error!
4286 sub vm_deviceunplug {
4287     my ($vmid, $conf, $deviceid) = @_;
4288
4289     my $devices_list = vm_devices_list($vmid);
4290     return 1 if !defined($devices_list->{$deviceid});
4291
4292     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4293     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4294
4295     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4296         qemu_devicedel($vmid, $deviceid);
4297     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4298         qemu_devicedel($vmid, $deviceid);
4299         qemu_devicedelverify($vmid, $deviceid);
4300     } elsif ($deviceid =~ m/^usb\d+$/) {
4301         qemu_devicedel($vmid, $deviceid);
4302         qemu_devicedelverify($vmid, $deviceid);
4303     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4304         my $device = parse_drive($deviceid, $conf->{$deviceid});
4305
4306         qemu_devicedel($vmid, $deviceid);
4307         qemu_devicedelverify($vmid, $deviceid);
4308         qemu_drivedel($vmid, $deviceid);
4309         qemu_iothread_del($vmid, $deviceid, $device);
4310     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4311         qemu_devicedel($vmid, $deviceid);
4312         qemu_devicedelverify($vmid, $deviceid);
4313     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4314         my $device = parse_drive($deviceid, $conf->{$deviceid});
4315
4316         qemu_devicedel($vmid, $deviceid);
4317         qemu_devicedelverify($vmid, $deviceid);
4318         qemu_drivedel($vmid, $deviceid);
4319         qemu_deletescsihw($conf, $vmid, $deviceid);
4320
4321         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4322             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4323     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4324         qemu_devicedel($vmid, $deviceid);
4325         qemu_devicedelverify($vmid, $deviceid);
4326         qemu_netdevdel($vmid, $deviceid);
4327     } else {
4328         die "can't unplug device '$deviceid'\n";
4329     }
4330
4331     return 1;
4332 }
4333
4334 sub qemu_spice_usbredir_chardev_add {
4335     my ($vmid, $id) = @_;
4336
4337     mon_cmd($vmid, "chardev-add" , (
4338         id => $id,
4339         backend => {
4340             type => 'spicevmc',
4341             data => {
4342                 type => "usbredir",
4343             },
4344         },
4345     ));
4346 }
4347
4348 sub qemu_iothread_add {
4349     my ($vmid, $deviceid, $device) = @_;
4350
4351     if ($device->{iothread}) {
4352         my $iothreads = vm_iothreads_list($vmid);
4353         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4354     }
4355 }
4356
4357 sub qemu_iothread_del {
4358     my ($vmid, $deviceid, $device) = @_;
4359
4360     if ($device->{iothread}) {
4361         my $iothreads = vm_iothreads_list($vmid);
4362         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4363     }
4364 }
4365
4366 sub qemu_driveadd {
4367     my ($storecfg, $vmid, $device) = @_;
4368
4369     my $kvmver = get_running_qemu_version($vmid);
4370     my $io_uring = min_version($kvmver, 6, 0);
4371     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4372     $drive =~ s/\\/\\\\/g;
4373     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4374
4375     # If the command succeeds qemu prints: "OK"
4376     return 1 if $ret =~ m/OK/s;
4377
4378     die "adding drive failed: $ret\n";
4379 }
4380
4381 sub qemu_drivedel {
4382     my ($vmid, $deviceid) = @_;
4383
4384     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4385     $ret =~ s/^\s+//;
4386
4387     return 1 if $ret eq "";
4388
4389     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4390     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4391
4392     die "deleting drive $deviceid failed : $ret\n";
4393 }
4394
4395 sub qemu_deviceaddverify {
4396     my ($vmid, $deviceid) = @_;
4397
4398     for (my $i = 0; $i <= 5; $i++) {
4399          my $devices_list = vm_devices_list($vmid);
4400          return 1 if defined($devices_list->{$deviceid});
4401          sleep 1;
4402     }
4403
4404     die "error on hotplug device '$deviceid'\n";
4405 }
4406
4407
4408 sub qemu_devicedelverify {
4409     my ($vmid, $deviceid) = @_;
4410
4411     # need to verify that the device is correctly removed as device_del
4412     # is async and empty return is not reliable
4413
4414     for (my $i = 0; $i <= 5; $i++) {
4415          my $devices_list = vm_devices_list($vmid);
4416          return 1 if !defined($devices_list->{$deviceid});
4417          sleep 1;
4418     }
4419
4420     die "error on hot-unplugging device '$deviceid'\n";
4421 }
4422
4423 sub qemu_findorcreatescsihw {
4424     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4425
4426     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4427
4428     my $scsihwid="$controller_prefix$controller";
4429     my $devices_list = vm_devices_list($vmid);
4430
4431     if (!defined($devices_list->{$scsihwid})) {
4432         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4433     }
4434
4435     return 1;
4436 }
4437
4438 sub qemu_deletescsihw {
4439     my ($conf, $vmid, $opt) = @_;
4440
4441     my $device = parse_drive($opt, $conf->{$opt});
4442
4443     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4444         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4445         return 1;
4446     }
4447
4448     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4449
4450     my $devices_list = vm_devices_list($vmid);
4451     foreach my $opt (keys %{$devices_list}) {
4452         if (is_valid_drivename($opt)) {
4453             my $drive = parse_drive($opt, $conf->{$opt});
4454             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4455                 return 1;
4456             }
4457         }
4458     }
4459
4460     my $scsihwid="scsihw$controller";
4461
4462     vm_deviceunplug($vmid, $conf, $scsihwid);
4463
4464     return 1;
4465 }
4466
4467 sub qemu_add_pci_bridge {
4468     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4469
4470     my $bridges = {};
4471
4472     my $bridgeid;
4473
4474     print_pci_addr($device, $bridges, $arch, $machine_type);
4475
4476     while (my ($k, $v) = each %$bridges) {
4477         $bridgeid = $k;
4478     }
4479     return 1 if !defined($bridgeid) || $bridgeid < 1;
4480
4481     my $bridge = "pci.$bridgeid";
4482     my $devices_list = vm_devices_list($vmid);
4483
4484     if (!defined($devices_list->{$bridge})) {
4485         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4486     }
4487
4488     return 1;
4489 }
4490
4491 sub qemu_set_link_status {
4492     my ($vmid, $device, $up) = @_;
4493
4494     mon_cmd($vmid, "set_link", name => $device,
4495                up => $up ? JSON::true : JSON::false);
4496 }
4497
4498 sub qemu_netdevadd {
4499     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4500
4501     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4502     my %options =  split(/[=,]/, $netdev);
4503
4504     if (defined(my $vhost = $options{vhost})) {
4505         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4506     }
4507
4508     if (defined(my $queues = $options{queues})) {
4509         $options{queues} = $queues + 0;
4510     }
4511
4512     mon_cmd($vmid, "netdev_add",  %options);
4513     return 1;
4514 }
4515
4516 sub qemu_netdevdel {
4517     my ($vmid, $deviceid) = @_;
4518
4519     mon_cmd($vmid, "netdev_del", id => $deviceid);
4520 }
4521
4522 sub qemu_usb_hotplug {
4523     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4524
4525     return if !$device;
4526
4527     # remove the old one first
4528     vm_deviceunplug($vmid, $conf, $deviceid);
4529
4530     # check if xhci controller is necessary and available
4531     my $devicelist = vm_devices_list($vmid);
4532
4533     if (!$devicelist->{xhci}) {
4534         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4535         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4536     }
4537
4538     # add the new one
4539     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4540 }
4541
4542 sub qemu_cpu_hotplug {
4543     my ($vmid, $conf, $vcpus) = @_;
4544
4545     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4546
4547     my $sockets = 1;
4548     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4549     $sockets = $conf->{sockets} if  $conf->{sockets};
4550     my $cores = $conf->{cores} || 1;
4551     my $maxcpus = $sockets * $cores;
4552
4553     $vcpus = $maxcpus if !$vcpus;
4554
4555     die "you can't add more vcpus than maxcpus\n"
4556         if $vcpus > $maxcpus;
4557
4558     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4559
4560     if ($vcpus < $currentvcpus) {
4561
4562         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4563
4564             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4565                 qemu_devicedel($vmid, "cpu$i");
4566                 my $retry = 0;
4567                 my $currentrunningvcpus = undef;
4568                 while (1) {
4569                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4570                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4571                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4572                     $retry++;
4573                     sleep 1;
4574                 }
4575                 #update conf after each succesfull cpu unplug
4576                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4577                 PVE::QemuConfig->write_config($vmid, $conf);
4578             }
4579         } else {
4580             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4581         }
4582
4583         return;
4584     }
4585
4586     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4587     die "vcpus in running vm does not match its configuration\n"
4588         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4589
4590     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4591
4592         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4593             my $cpustr = print_cpu_device($conf, $i);
4594             qemu_deviceadd($vmid, $cpustr);
4595
4596             my $retry = 0;
4597             my $currentrunningvcpus = undef;
4598             while (1) {
4599                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4600                 last if scalar(@{$currentrunningvcpus}) == $i;
4601                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4602                 sleep 1;
4603                 $retry++;
4604             }
4605             #update conf after each succesfull cpu hotplug
4606             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4607             PVE::QemuConfig->write_config($vmid, $conf);
4608         }
4609     } else {
4610
4611         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4612             mon_cmd($vmid, "cpu-add", id => int($i));
4613         }
4614     }
4615 }
4616
4617 sub qemu_block_set_io_throttle {
4618     my ($vmid, $deviceid,
4619         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4620         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4621         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4622         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4623
4624     return if !check_running($vmid) ;
4625
4626     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4627         bps => int($bps),
4628         bps_rd => int($bps_rd),
4629         bps_wr => int($bps_wr),
4630         iops => int($iops),
4631         iops_rd => int($iops_rd),
4632         iops_wr => int($iops_wr),
4633         bps_max => int($bps_max),
4634         bps_rd_max => int($bps_rd_max),
4635         bps_wr_max => int($bps_wr_max),
4636         iops_max => int($iops_max),
4637         iops_rd_max => int($iops_rd_max),
4638         iops_wr_max => int($iops_wr_max),
4639         bps_max_length => int($bps_max_length),
4640         bps_rd_max_length => int($bps_rd_max_length),
4641         bps_wr_max_length => int($bps_wr_max_length),
4642         iops_max_length => int($iops_max_length),
4643         iops_rd_max_length => int($iops_rd_max_length),
4644         iops_wr_max_length => int($iops_wr_max_length),
4645     );
4646
4647 }
4648
4649 sub qemu_block_resize {
4650     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4651
4652     my $running = check_running($vmid);
4653
4654     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4655
4656     return if !$running;
4657
4658     my $padding = (1024 - $size % 1024) % 1024;
4659     $size = $size + $padding;
4660
4661     mon_cmd(
4662         $vmid,
4663         "block_resize",
4664         device => $deviceid,
4665         size => int($size),
4666         timeout => 60,
4667     );
4668 }
4669
4670 sub qemu_volume_snapshot {
4671     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4672
4673     my $running = check_running($vmid);
4674
4675     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4676         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4677     } else {
4678         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4679     }
4680 }
4681
4682 sub qemu_volume_snapshot_delete {
4683     my ($vmid, $storecfg, $volid, $snap) = @_;
4684
4685     my $running = check_running($vmid);
4686     my $attached_deviceid;
4687
4688     if ($running) {
4689         my $conf = PVE::QemuConfig->load_config($vmid);
4690         PVE::QemuConfig->foreach_volume($conf, sub {
4691             my ($ds, $drive) = @_;
4692             $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4693         });
4694     }
4695
4696     if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4697         mon_cmd(
4698             $vmid,
4699             'blockdev-snapshot-delete-internal-sync',
4700             device => $attached_deviceid,
4701             name => $snap,
4702         );
4703     } else {
4704         PVE::Storage::volume_snapshot_delete(
4705             $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4706     }
4707 }
4708
4709 sub set_migration_caps {
4710     my ($vmid, $savevm) = @_;
4711
4712     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4713
4714     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4715     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4716
4717     my $cap_ref = [];
4718
4719     my $enabled_cap = {
4720         "auto-converge" => 1,
4721         "xbzrle" => 1,
4722         "x-rdma-pin-all" => 0,
4723         "zero-blocks" => 0,
4724         "compress" => 0,
4725         "dirty-bitmaps" => $dirty_bitmaps,
4726     };
4727
4728     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4729
4730     for my $supported_capability (@$supported_capabilities) {
4731         push @$cap_ref, {
4732             capability => $supported_capability->{capability},
4733             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4734         };
4735     }
4736
4737     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4738 }
4739
4740 sub foreach_volid {
4741     my ($conf, $func, @param) = @_;
4742
4743     my $volhash = {};
4744
4745     my $test_volid = sub {
4746         my ($key, $drive, $snapname, $pending) = @_;
4747
4748         my $volid = $drive->{file};
4749         return if !$volid;
4750
4751         $volhash->{$volid}->{cdrom} //= 1;
4752         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4753
4754         my $replicate = $drive->{replicate} // 1;
4755         $volhash->{$volid}->{replicate} //= 0;
4756         $volhash->{$volid}->{replicate} = 1 if $replicate;
4757
4758         $volhash->{$volid}->{shared} //= 0;
4759         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4760
4761         $volhash->{$volid}->{is_unused} //= 0;
4762         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4763
4764         $volhash->{$volid}->{is_attached} //= 0;
4765         $volhash->{$volid}->{is_attached} = 1
4766             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4767
4768         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4769             if defined($snapname);
4770
4771         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4772
4773         my $size = $drive->{size};
4774         $volhash->{$volid}->{size} //= $size if $size;
4775
4776         $volhash->{$volid}->{is_vmstate} //= 0;
4777         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4778
4779         $volhash->{$volid}->{is_tpmstate} //= 0;
4780         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4781
4782         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4783     };
4784
4785     my $include_opts = {
4786         extra_keys => ['vmstate'],
4787         include_unused => 1,
4788     };
4789
4790     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4791
4792     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4793         if defined($conf->{pending}) && $conf->{pending}->%*;
4794
4795     foreach my $snapname (keys %{$conf->{snapshots}}) {
4796         my $snap = $conf->{snapshots}->{$snapname};
4797         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4798     }
4799
4800     foreach my $volid (keys %$volhash) {
4801         &$func($volid, $volhash->{$volid}, @param);
4802     }
4803 }
4804
4805 my $fast_plug_option = {
4806     'description' => 1,
4807     'hookscript' => 1,
4808     'lock' => 1,
4809     'migrate_downtime' => 1,
4810     'migrate_speed' => 1,
4811     'name' => 1,
4812     'onboot' => 1,
4813     'protection' => 1,
4814     'shares' => 1,
4815     'startup' => 1,
4816     'tags' => 1,
4817     'vmstatestorage' => 1,
4818 };
4819
4820 for my $opt (keys %$confdesc_cloudinit) {
4821     $fast_plug_option->{$opt} = 1;
4822 };
4823
4824 # hotplug changes in [PENDING]
4825 # $selection hash can be used to only apply specified options, for
4826 # example: { cores => 1 } (only apply changed 'cores')
4827 # $errors ref is used to return error messages
4828 sub vmconfig_hotplug_pending {
4829     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4830
4831     my $defaults = load_defaults();
4832     my $arch = get_vm_arch($conf);
4833     my $machine_type = get_vm_machine($conf, undef, $arch);
4834
4835     # commit values which do not have any impact on running VM first
4836     # Note: those option cannot raise errors, we we do not care about
4837     # $selection and always apply them.
4838
4839     my $add_error = sub {
4840         my ($opt, $msg) = @_;
4841         $errors->{$opt} = "hotplug problem - $msg";
4842     };
4843
4844     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4845
4846     my $cloudinit_record_changed = sub {
4847         my ($conf, $opt, $old, $new) = @_;
4848         return if !$cloudinit_pending_properties->{$opt};
4849
4850         my $ci = ($conf->{cloudinit} //= {});
4851
4852         my $recorded = $ci->{$opt};
4853         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4854
4855         if (defined($new)) {
4856             if (defined($old)) {
4857                 # an existing value is being modified
4858                 if (defined($recorded)) {
4859                     # the value was already not in sync
4860                     if ($new eq $recorded) {
4861                         # a value is being reverted to the cloud-init state:
4862                         delete $ci->{$opt};
4863                         delete $added{$opt};
4864                     } else {
4865                         # the value was changed multiple times, do nothing
4866                     }
4867                 } elsif ($added{$opt}) {
4868                     # the value had been marked as added and is being changed, do nothing
4869                 } else {
4870                     # the value is new, record it:
4871                     $ci->{$opt} = $old;
4872                 }
4873             } else {
4874                 # a new value is being added
4875                 if (defined($recorded)) {
4876                     # it was already not in sync
4877                     if ($new eq $recorded) {
4878                         # a value is being reverted to the cloud-init state:
4879                         delete $ci->{$opt};
4880                         delete $added{$opt};
4881                     } else {
4882                         # the value had temporarily been removed, do nothing
4883                     }
4884                 } elsif ($added{$opt}) {
4885                     # the value had been marked as added already, do nothing
4886                 } else {
4887                     # the value is new, add it
4888                     $added{$opt} = 1;
4889                 }
4890             }
4891         } elsif (!defined($old)) {
4892             # a non-existent value is being removed? ignore...
4893         } else {
4894             # a value is being deleted
4895             if (defined($recorded)) {
4896                 # a value was already recorded, just keep it
4897             } elsif ($added{$opt}) {
4898                 # the value was marked as added, remove it
4899                 delete $added{$opt};
4900             } else {
4901                 # a previously unrecorded value is being removed, record the old value:
4902                 $ci->{$opt} = $old;
4903             }
4904         }
4905
4906         my $added = join(',', sort keys %added);
4907         $ci->{added} = $added if length($added);
4908     };
4909
4910     my $changes = 0;
4911     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4912         if ($fast_plug_option->{$opt}) {
4913             my $new = delete $conf->{pending}->{$opt};
4914             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4915             $conf->{$opt} = $new;
4916             $changes = 1;
4917         }
4918     }
4919
4920     if ($changes) {
4921         PVE::QemuConfig->write_config($vmid, $conf);
4922     }
4923
4924     my $ostype = $conf->{ostype};
4925     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4926     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4927     my $usb_hotplug = $hotplug_features->{usb}
4928         && min_version($version, 7, 1)
4929         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4930
4931     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4932     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4933
4934     foreach my $opt (sort keys %$pending_delete_hash) {
4935         next if $selection && !$selection->{$opt};
4936         my $force = $pending_delete_hash->{$opt}->{force};
4937         eval {
4938             if ($opt eq 'hotplug') {
4939                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4940             } elsif ($opt eq 'tablet') {
4941                 die "skip\n" if !$hotplug_features->{usb};
4942                 if ($defaults->{tablet}) {
4943                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4944                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4945                         if $arch eq 'aarch64';
4946                 } else {
4947                     vm_deviceunplug($vmid, $conf, 'tablet');
4948                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4949                 }
4950             } elsif ($opt =~ m/^usb(\d+)$/) {
4951                 my $index = $1;
4952                 die "skip\n" if !$usb_hotplug;
4953                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4954                 vm_deviceunplug($vmid, $conf, $opt);
4955             } elsif ($opt eq 'vcpus') {
4956                 die "skip\n" if !$hotplug_features->{cpu};
4957                 qemu_cpu_hotplug($vmid, $conf, undef);
4958             } elsif ($opt eq 'balloon') {
4959                 # enable balloon device is not hotpluggable
4960                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4961                 # here we reset the ballooning value to memory
4962                 my $balloon = get_current_memory($conf->{memory});
4963                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4964             } elsif ($fast_plug_option->{$opt}) {
4965                 # do nothing
4966             } elsif ($opt =~ m/^net(\d+)$/) {
4967                 die "skip\n" if !$hotplug_features->{network};
4968                 vm_deviceunplug($vmid, $conf, $opt);
4969                 if($have_sdn) {
4970                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
4971                     PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
4972                 }
4973             } elsif (is_valid_drivename($opt)) {
4974                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4975                 vm_deviceunplug($vmid, $conf, $opt);
4976                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4977             } elsif ($opt =~ m/^memory$/) {
4978                 die "skip\n" if !$hotplug_features->{memory};
4979                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
4980             } elsif ($opt eq 'cpuunits') {
4981                 $cgroup->change_cpu_shares(undef);
4982             } elsif ($opt eq 'cpulimit') {
4983                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4984             } else {
4985                 die "skip\n";
4986             }
4987         };
4988         if (my $err = $@) {
4989             &$add_error($opt, $err) if $err ne "skip\n";
4990         } else {
4991             my $old = delete $conf->{$opt};
4992             $cloudinit_record_changed->($conf, $opt, $old, undef);
4993             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4994         }
4995     }
4996
4997     my $cloudinit_opt;
4998     foreach my $opt (keys %{$conf->{pending}}) {
4999         next if $selection && !$selection->{$opt};
5000         my $value = $conf->{pending}->{$opt};
5001         eval {
5002             if ($opt eq 'hotplug') {
5003                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5004                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
5005             } elsif ($opt eq 'tablet') {
5006                 die "skip\n" if !$hotplug_features->{usb};
5007                 if ($value == 1) {
5008                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5009                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5010                         if $arch eq 'aarch64';
5011                 } elsif ($value == 0) {
5012                     vm_deviceunplug($vmid, $conf, 'tablet');
5013                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5014                 }
5015             } elsif ($opt =~ m/^usb(\d+)$/) {
5016                 my $index = $1;
5017                 die "skip\n" if !$usb_hotplug;
5018                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5019                 my $id = $opt;
5020                 if ($d->{host} =~ m/^spice$/i)  {
5021                     $id = "usbredirdev$index";
5022                 }
5023                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5024             } elsif ($opt eq 'vcpus') {
5025                 die "skip\n" if !$hotplug_features->{cpu};
5026                 qemu_cpu_hotplug($vmid, $conf, $value);
5027             } elsif ($opt eq 'balloon') {
5028                 # enable/disable balloning device is not hotpluggable
5029                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5030                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5031                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5032
5033                 # allow manual ballooning if shares is set to zero
5034                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5035                     my $memory = get_current_memory($conf->{memory});
5036                     my $balloon = $conf->{pending}->{balloon} || $memory;
5037                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5038                 }
5039             } elsif ($opt =~ m/^net(\d+)$/) {
5040                 # some changes can be done without hotplug
5041                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5042                                     $vmid, $opt, $value, $arch, $machine_type);
5043             } elsif (is_valid_drivename($opt)) {
5044                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5045                 # some changes can be done without hotplug
5046                 my $drive = parse_drive($opt, $value);
5047                 if (drive_is_cloudinit($drive)) {
5048                     $cloudinit_opt = [$opt, $drive];
5049                     # apply all the other changes first, then generate the cloudinit disk
5050                     die "skip\n";
5051                 }
5052                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5053                                      $vmid, $opt, $value, $arch, $machine_type);
5054             } elsif ($opt =~ m/^memory$/) { #dimms
5055                 die "skip\n" if !$hotplug_features->{memory};
5056                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5057             } elsif ($opt eq 'cpuunits') {
5058                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5059                 $cgroup->change_cpu_shares($new_cpuunits);
5060             } elsif ($opt eq 'cpulimit') {
5061                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5062                 $cgroup->change_cpu_quota($cpulimit, 100000);
5063             } elsif ($opt eq 'agent') {
5064                 vmconfig_update_agent($conf, $opt, $value);
5065             } else {
5066                 die "skip\n";  # skip non-hot-pluggable options
5067             }
5068         };
5069         if (my $err = $@) {
5070             &$add_error($opt, $err) if $err ne "skip\n";
5071         } else {
5072             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5073             $conf->{$opt} = $value;
5074             delete $conf->{pending}->{$opt};
5075         }
5076     }
5077
5078     if (defined($cloudinit_opt)) {
5079         my ($opt, $drive) = @$cloudinit_opt;
5080         my $value = $conf->{pending}->{$opt};
5081         eval {
5082             my $temp = {%$conf, $opt => $value};
5083             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5084             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5085                                  $vmid, $opt, $value, $arch, $machine_type);
5086         };
5087         if (my $err = $@) {
5088             &$add_error($opt, $err) if $err ne "skip\n";
5089         } else {
5090             $conf->{$opt} = $value;
5091             delete $conf->{pending}->{$opt};
5092         }
5093     }
5094
5095     # unplug xhci controller if no usb device is left
5096     if ($usb_hotplug) {
5097         my $has_usb = 0;
5098         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5099             next if !defined($conf->{"usb$i"});
5100             $has_usb = 1;
5101             last;
5102         }
5103         if (!$has_usb) {
5104             vm_deviceunplug($vmid, $conf, 'xhci');
5105         }
5106     }
5107
5108     PVE::QemuConfig->write_config($vmid, $conf);
5109
5110     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5111         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5112     }
5113 }
5114
5115 sub try_deallocate_drive {
5116     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5117
5118     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5119         my $volid = $drive->{file};
5120         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5121             my $sid = PVE::Storage::parse_volume_id($volid);
5122             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5123
5124             # check if the disk is really unused
5125             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5126                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5127             PVE::Storage::vdisk_free($storecfg, $volid);
5128             return 1;
5129         } else {
5130             # If vm is not owner of this disk remove from config
5131             return 1;
5132         }
5133     }
5134
5135     return;
5136 }
5137
5138 sub vmconfig_delete_or_detach_drive {
5139     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5140
5141     my $drive = parse_drive($opt, $conf->{$opt});
5142
5143     my $rpcenv = PVE::RPCEnvironment::get();
5144     my $authuser = $rpcenv->get_user();
5145
5146     if ($force) {
5147         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5148         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5149     } else {
5150         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5151     }
5152 }
5153
5154
5155
5156 sub vmconfig_apply_pending {
5157     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5158
5159     return if !scalar(keys %{$conf->{pending}});
5160
5161     my $add_apply_error = sub {
5162         my ($opt, $msg) = @_;
5163         my $err_msg = "unable to apply pending change $opt : $msg";
5164         $errors->{$opt} = $err_msg;
5165         warn $err_msg;
5166     };
5167
5168     # cold plug
5169
5170     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5171     foreach my $opt (sort keys %$pending_delete_hash) {
5172         my $force = $pending_delete_hash->{$opt}->{force};
5173         eval {
5174             if ($opt =~ m/^unused/) {
5175                 die "internal error";
5176             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5177                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5178             } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5179                 if($have_sdn) {
5180                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5181                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5182                     warn if $@;
5183                 }
5184             }
5185         };
5186         if (my $err = $@) {
5187             $add_apply_error->($opt, $err);
5188         } else {
5189             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5190             delete $conf->{$opt};
5191         }
5192     }
5193
5194     PVE::QemuConfig->cleanup_pending($conf);
5195
5196     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5197
5198     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5199         next if $opt eq 'delete'; # just to be sure
5200         eval {
5201             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5202                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5203             } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5204                 if($have_sdn) {
5205                     my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5206                     if ($conf->{$opt}){
5207                         my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5208
5209                         if ($old_net->{bridge} ne $new_net->{bridge} ||
5210                             $old_net->{macaddr} ne $new_net->{macaddr}) {
5211                             PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5212                         }
5213                    }
5214                    #fixme: reuse ip if mac change && same bridge
5215                    PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5216                 }
5217             }
5218         };
5219         if (my $err = $@) {
5220             $add_apply_error->($opt, $err);
5221         } else {
5222
5223             if (is_valid_drivename($opt)) {
5224                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5225                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5226             }
5227
5228             $conf->{$opt} = delete $conf->{pending}->{$opt};
5229         }
5230     }
5231
5232     # write all changes at once to avoid unnecessary i/o
5233     PVE::QemuConfig->write_config($vmid, $conf);
5234     if ($generate_cloudinit) {
5235         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5236             # After successful generation and if there were changes to be applied, update the
5237             # config to drop the {cloudinit} entry.
5238             PVE::QemuConfig->write_config($vmid, $conf);
5239         }
5240     }
5241 }
5242
5243 sub vmconfig_update_net {
5244     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5245
5246     my $newnet = parse_net($value);
5247
5248     if ($conf->{$opt}) {
5249         my $oldnet = parse_net($conf->{$opt});
5250
5251         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5252             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5253             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5254             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5255             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5256
5257             # for non online change, we try to hot-unplug
5258             die "skip\n" if !$hotplug;
5259             vm_deviceunplug($vmid, $conf, $opt);
5260
5261             if($have_sdn) {
5262                 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5263             }
5264
5265         } else {
5266
5267             die "internal error" if $opt !~ m/net(\d+)/;
5268             my $iface = "tap${vmid}i$1";
5269
5270             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5271                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5272                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5273                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5274                 PVE::Network::tap_unplug($iface);
5275
5276                 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5277                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5278                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5279                     qemu_set_link_status($vmid, $opt, 0);
5280                 }
5281
5282                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5283                     if ($have_sdn) {
5284                         PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5285                         PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5286                     }
5287                 }
5288
5289                 if ($have_sdn) {
5290                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5291                 } else {
5292                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5293                 }
5294
5295                 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5296                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5297                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5298                     qemu_set_link_status($vmid, $opt, 1);
5299                 }
5300
5301             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5302                 # Rate can be applied on its own but any change above needs to
5303                 # include the rate in tap_plug since OVS resets everything.
5304                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5305             }
5306
5307             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5308                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5309             }
5310
5311             return 1;
5312         }
5313     }
5314
5315     if ($hotplug) {
5316         if ($have_sdn) {
5317             PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5318             PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5319         }
5320         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5321     } else {
5322         die "skip\n";
5323     }
5324 }
5325
5326 sub vmconfig_update_agent {
5327     my ($conf, $opt, $value) = @_;
5328
5329     die "skip\n" if !$conf->{$opt};
5330
5331     my $hotplug_options = { fstrim_cloned_disks => 1 };
5332
5333     my $old_agent = parse_guest_agent($conf);
5334     my $agent = parse_guest_agent({$opt => $value});
5335
5336     for my $option (keys %$agent) { # added/changed options
5337         next if defined($hotplug_options->{$option});
5338         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5339     }
5340
5341     for my $option (keys %$old_agent) { # removed options
5342         next if defined($hotplug_options->{$option});
5343         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5344     }
5345
5346     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5347 }
5348
5349 sub vmconfig_update_disk {
5350     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5351
5352     my $drive = parse_drive($opt, $value);
5353
5354     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5355         my $media = $drive->{media} || 'disk';
5356         my $oldmedia = $old_drive->{media} || 'disk';
5357         die "unable to change media type\n" if $media ne $oldmedia;
5358
5359         if (!drive_is_cdrom($old_drive)) {
5360
5361             if ($drive->{file} ne $old_drive->{file}) {
5362
5363                 die "skip\n" if !$hotplug;
5364
5365                 # unplug and register as unused
5366                 vm_deviceunplug($vmid, $conf, $opt);
5367                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5368
5369             } else {
5370                 # update existing disk
5371
5372                 # skip non hotpluggable value
5373                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5374                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5375                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5376                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5377                     safe_string_ne($drive->{product}, $old_drive->{product}) ||
5378                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5379                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5380                     safe_string_ne($drive->{vendor}, $old_drive->{vendor}) ||
5381                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5382                     die "skip\n";
5383                 }
5384
5385                 # apply throttle
5386                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5387                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5388                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5389                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5390                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5391                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5392                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5393                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5394                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5395                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5396                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5397                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5398                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5399                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5400                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5401                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5402                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5403                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5404
5405                     qemu_block_set_io_throttle(
5406                         $vmid,"drive-$opt",
5407                         ($drive->{mbps} || 0)*1024*1024,
5408                         ($drive->{mbps_rd} || 0)*1024*1024,
5409                         ($drive->{mbps_wr} || 0)*1024*1024,
5410                          $drive->{iops} || 0,
5411                          $drive->{iops_rd} || 0,
5412                          $drive->{iops_wr} || 0,
5413                         ($drive->{mbps_max} || 0)*1024*1024,
5414                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5415                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5416                          $drive->{iops_max} || 0,
5417                          $drive->{iops_rd_max} || 0,
5418                          $drive->{iops_wr_max} || 0,
5419                          $drive->{bps_max_length} || 1,
5420                          $drive->{bps_rd_max_length} || 1,
5421                          $drive->{bps_wr_max_length} || 1,
5422                          $drive->{iops_max_length} || 1,
5423                          $drive->{iops_rd_max_length} || 1,
5424                          $drive->{iops_wr_max_length} || 1,
5425                     );
5426
5427                 }
5428
5429                 return 1;
5430             }
5431
5432         } else { # cdrom
5433
5434             if ($drive->{file} eq 'none') {
5435                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5436                 if (drive_is_cloudinit($old_drive)) {
5437                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5438                 }
5439             } else {
5440                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5441
5442                 # force eject if locked
5443                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5444
5445                 if ($path) {
5446                     mon_cmd($vmid, "blockdev-change-medium",
5447                         id => "$opt", filename => "$path");
5448                 }
5449             }
5450
5451             return 1;
5452         }
5453     }
5454
5455     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5456     # hotplug new disks
5457     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5458     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5459 }
5460
5461 sub vmconfig_update_cloudinit_drive {
5462     my ($storecfg, $conf, $vmid) = @_;
5463
5464     my $cloudinit_ds = undef;
5465     my $cloudinit_drive = undef;
5466
5467     PVE::QemuConfig->foreach_volume($conf, sub {
5468         my ($ds, $drive) = @_;
5469         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5470             $cloudinit_ds = $ds;
5471             $cloudinit_drive = $drive;
5472         }
5473     });
5474
5475     return if !$cloudinit_drive;
5476
5477     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5478         PVE::QemuConfig->write_config($vmid, $conf);
5479     }
5480
5481     my $running = PVE::QemuServer::check_running($vmid);
5482
5483     if ($running) {
5484         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5485         if ($path) {
5486             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5487             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5488         }
5489     }
5490 }
5491
5492 # called in locked context by incoming migration
5493 sub vm_migrate_get_nbd_disks {
5494     my ($storecfg, $conf, $replicated_volumes) = @_;
5495
5496     my $local_volumes = {};
5497     PVE::QemuConfig->foreach_volume($conf, sub {
5498         my ($ds, $drive) = @_;
5499
5500         return if drive_is_cdrom($drive);
5501         return if $ds eq 'tpmstate0';
5502
5503         my $volid = $drive->{file};
5504
5505         return if !$volid;
5506
5507         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5508
5509         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5510         return if $scfg->{shared};
5511
5512         my $format = qemu_img_format($scfg, $volname);
5513
5514         # replicated disks re-use existing state via bitmap
5515         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5516         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5517     });
5518     return $local_volumes;
5519 }
5520
5521 # called in locked context by incoming migration
5522 sub vm_migrate_alloc_nbd_disks {
5523     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5524
5525     my $nbd = {};
5526     foreach my $opt (sort keys %$source_volumes) {
5527         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5528
5529         if ($use_existing) {
5530             $nbd->{$opt}->{drivestr} = print_drive($drive);
5531             $nbd->{$opt}->{volid} = $volid;
5532             $nbd->{$opt}->{replicated} = 1;
5533             next;
5534         }
5535
5536         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5537
5538         # order of precedence, filtered by whether storage supports it:
5539         # 1. explicit requested format
5540         # 2. default format of storage
5541         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5542         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5543
5544         my $size = $drive->{size} / 1024;
5545         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5546         my $newdrive = $drive;
5547         $newdrive->{format} = $format;
5548         $newdrive->{file} = $newvolid;
5549         my $drivestr = print_drive($newdrive);
5550         $nbd->{$opt}->{drivestr} = $drivestr;
5551         $nbd->{$opt}->{volid} = $newvolid;
5552     }
5553
5554     return $nbd;
5555 }
5556
5557 # see vm_start_nolock for parameters, additionally:
5558 # migrate_opts:
5559 #   storagemap = parsed storage map for allocating NBD disks
5560 sub vm_start {
5561     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5562
5563     return PVE::QemuConfig->lock_config($vmid, sub {
5564         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5565
5566         die "you can't start a vm if it's a template\n"
5567             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5568
5569         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5570         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5571
5572         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5573
5574         if ($has_backup_lock && $running) {
5575             # a backup is currently running, attempt to start the guest in the
5576             # existing QEMU instance
5577             return vm_resume($vmid);
5578         }
5579
5580         PVE::QemuConfig->check_lock($conf)
5581             if !($params->{skiplock} || $has_suspended_lock);
5582
5583         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5584
5585         die "VM $vmid already running\n" if $running;
5586
5587         if (my $storagemap = $migrate_opts->{storagemap}) {
5588             my $replicated = $migrate_opts->{replicated_volumes};
5589             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5590             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5591
5592             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5593                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5594             }
5595         }
5596
5597         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5598     });
5599 }
5600
5601
5602 # params:
5603 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5604 #   skiplock => 0/1, skip checking for config lock
5605 #   skiptemplate => 0/1, skip checking whether VM is template
5606 #   forcemachine => to force QEMU machine (rollback/migration)
5607 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5608 #   timeout => in seconds
5609 #   paused => start VM in paused state (backup)
5610 #   resume => resume from hibernation
5611 #   pbs-backing => {
5612 #      sata0 => {
5613 #         repository
5614 #         snapshot
5615 #         keyfile
5616 #         archive
5617 #      },
5618 #      virtio2 => ...
5619 #   }
5620 # migrate_opts:
5621 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5622 #   migratedfrom => source node
5623 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5624 #   network => CIDR of migration network
5625 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5626 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5627 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5628 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5629 #       contained in config
5630 sub vm_start_nolock {
5631     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5632
5633     my $statefile = $params->{statefile};
5634     my $resume = $params->{resume};
5635
5636     my $migratedfrom = $migrate_opts->{migratedfrom};
5637     my $migration_type = $migrate_opts->{type};
5638
5639     my $res = {};
5640
5641     # clean up leftover reboot request files
5642     eval { clear_reboot_request($vmid); };
5643     warn $@ if $@;
5644
5645     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5646         vmconfig_apply_pending($vmid, $conf, $storecfg);
5647         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5648     }
5649
5650     # don't regenerate the ISO if the VM is started as part of a live migration
5651     # this way we can reuse the old ISO with the correct config
5652     if (!$migratedfrom) {
5653         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5654             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5655             # $conf->{cloudinit}, so we could just not do this?
5656             # But we do it above, so for now let's be consistent.
5657             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5658         }
5659     }
5660
5661     # override offline migrated volumes, conf is out of date still
5662     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5663         for my $key (sort keys $offline_volumes->%*) {
5664             my $parsed = parse_drive($key, $conf->{$key});
5665             $parsed->{file} = $offline_volumes->{$key};
5666             $conf->{$key} = print_drive($parsed);
5667         }
5668     }
5669
5670     my $defaults = load_defaults();
5671
5672     # set environment variable useful inside network script
5673     # for remote migration the config is available on the target node!
5674     if (!$migrate_opts->{remote_node}) {
5675         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5676     }
5677
5678     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5679
5680     my $forcemachine = $params->{forcemachine};
5681     my $forcecpu = $params->{forcecpu};
5682     if ($resume) {
5683         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5684         $forcemachine = $conf->{runningmachine};
5685         $forcecpu = $conf->{runningcpu};
5686         print "Resuming suspended VM\n";
5687     }
5688
5689     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5690         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5691
5692     my $migration_ip;
5693     my $get_migration_ip = sub {
5694         my ($nodename) = @_;
5695
5696         return $migration_ip if defined($migration_ip);
5697
5698         my $cidr = $migrate_opts->{network};
5699
5700         if (!defined($cidr)) {
5701             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5702             $cidr = $dc_conf->{migration}->{network};
5703         }
5704
5705         if (defined($cidr)) {
5706             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5707
5708             die "could not get IP: no address configured on local " .
5709                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5710
5711             die "could not get IP: multiple addresses configured on local " .
5712                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5713
5714             $migration_ip = @$ips[0];
5715         }
5716
5717         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5718             if !defined($migration_ip);
5719
5720         return $migration_ip;
5721     };
5722
5723     if ($statefile) {
5724         if ($statefile eq 'tcp') {
5725             my $migrate = $res->{migrate} = { proto => 'tcp' };
5726             $migrate->{addr} = "localhost";
5727             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5728             my $nodename = nodename();
5729
5730             if (!defined($migration_type)) {
5731                 if (defined($datacenterconf->{migration}->{type})) {
5732                     $migration_type = $datacenterconf->{migration}->{type};
5733                 } else {
5734                     $migration_type = 'secure';
5735                 }
5736             }
5737
5738             if ($migration_type eq 'insecure') {
5739                 $migrate->{addr} = $get_migration_ip->($nodename);
5740                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5741             }
5742
5743             # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5744             # via QMP later
5745             push @$cmd, '-incoming', 'defer';
5746             push @$cmd, '-S';
5747
5748         } elsif ($statefile eq 'unix') {
5749             # should be default for secure migrations as a ssh TCP forward
5750             # tunnel is not deterministic reliable ready and fails regurarly
5751             # to set up in time, so use UNIX socket forwards
5752             my $migrate = $res->{migrate} = { proto => 'unix' };
5753             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5754             unlink $migrate->{addr};
5755
5756             $migrate->{uri} = "unix:$migrate->{addr}";
5757             push @$cmd, '-incoming', $migrate->{uri};
5758             push @$cmd, '-S';
5759
5760         } elsif (-e $statefile) {
5761             push @$cmd, '-loadstate', $statefile;
5762         } else {
5763             my $statepath = PVE::Storage::path($storecfg, $statefile);
5764             push @$vollist, $statefile;
5765             push @$cmd, '-loadstate', $statepath;
5766         }
5767     } elsif ($params->{paused}) {
5768         push @$cmd, '-S';
5769     }
5770
5771     my $memory = get_current_memory($conf->{memory});
5772     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5773
5774     my $pci_reserve_list = [];
5775     for my $device (values $pci_devices->%*) {
5776         next if $device->{mdev}; # we don't reserve for mdev devices
5777         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5778     }
5779
5780     # reserve all PCI IDs before actually doing anything with them
5781     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5782
5783     eval {
5784         my $uuid;
5785         for my $id (sort keys %$pci_devices) {
5786             my $d = $pci_devices->{$id};
5787             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5788
5789             my $chosen_mdev;
5790             for my $dev ($d->{ids}->@*) {
5791                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5792                 if ($d->{mdev}) {
5793                     warn $@ if $@;
5794                     $chosen_mdev = $info;
5795                     last if $chosen_mdev; # if successful, we're done
5796                 } else {
5797                     die $@ if $@;
5798                 }
5799             }
5800
5801             next if !$d->{mdev};
5802             die "could not create mediated device\n" if !defined($chosen_mdev);
5803
5804             # nvidia grid needs the uuid of the mdev as qemu parameter
5805             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5806                 if (defined($conf->{smbios1})) {
5807                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5808                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5809                 }
5810                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5811             }
5812         }
5813         push @$cmd, '-uuid', $uuid if defined($uuid);
5814     };
5815     if (my $err = $@) {
5816         eval { cleanup_pci_devices($vmid, $conf) };
5817         warn $@ if $@;
5818         die $err;
5819     }
5820
5821     PVE::Storage::activate_volumes($storecfg, $vollist);
5822
5823
5824     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5825     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5826     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5827     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5828     # timeout should be more than enough here...
5829     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5830
5831     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5832
5833     my %run_params = (
5834         timeout => $statefile ? undef : $start_timeout,
5835         umask => 0077,
5836         noerr => 1,
5837     );
5838
5839     # when migrating, prefix QEMU output so other side can pick up any
5840     # errors that might occur and show the user
5841     if ($migratedfrom) {
5842         $run_params{quiet} = 1;
5843         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5844     }
5845
5846     my %systemd_properties = (
5847         Slice => 'qemu.slice',
5848         KillMode => 'process',
5849         SendSIGKILL => 0,
5850         TimeoutStopUSec => ULONG_MAX, # infinity
5851     );
5852
5853     if (PVE::CGroup::cgroup_mode() == 2) {
5854         $systemd_properties{CPUWeight} = $cpuunits;
5855     } else {
5856         $systemd_properties{CPUShares} = $cpuunits;
5857     }
5858
5859     if (my $cpulimit = $conf->{cpulimit}) {
5860         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5861     }
5862     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5863
5864     my $run_qemu = sub {
5865         PVE::Tools::run_fork sub {
5866             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5867
5868             my $tpmpid;
5869             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5870                 # start the TPM emulator so QEMU can connect on start
5871                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5872             }
5873
5874             my $exitcode = run_command($cmd, %run_params);
5875             if ($exitcode) {
5876                 if ($tpmpid) {
5877                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5878                     kill 'TERM', $tpmpid;
5879                 }
5880                 die "QEMU exited with code $exitcode\n";
5881             }
5882         };
5883     };
5884
5885     if ($conf->{hugepages}) {
5886
5887         my $code = sub {
5888             my $hotplug_features =
5889                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5890             my $hugepages_topology =
5891                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5892
5893             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5894
5895             PVE::QemuServer::Memory::hugepages_mount();
5896             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5897
5898             eval { $run_qemu->() };
5899             if (my $err = $@) {
5900                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5901                     if !$conf->{keephugepages};
5902                 die $err;
5903             }
5904
5905             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5906                 if !$conf->{keephugepages};
5907         };
5908         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5909
5910     } else {
5911         eval { $run_qemu->() };
5912     }
5913
5914     if (my $err = $@) {
5915         # deactivate volumes if start fails
5916         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5917         warn $@ if $@;
5918         eval { cleanup_pci_devices($vmid, $conf) };
5919         warn $@ if $@;
5920
5921         die "start failed: $err";
5922     }
5923
5924     # re-reserve all PCI IDs now that we can know the actual VM PID
5925     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5926     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5927     warn $@ if $@;
5928
5929     if (defined(my $migrate = $res->{migrate})) {
5930         if ($migrate->{proto} eq 'tcp') {
5931             my $nodename = nodename();
5932             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5933             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5934             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5935             mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
5936         }
5937         print "migration listens on $migrate->{uri}\n";
5938     } elsif ($statefile) {
5939         eval { mon_cmd($vmid, "cont"); };
5940         warn $@ if $@;
5941     }
5942
5943     #start nbd server for storage migration
5944     if (my $nbd = $migrate_opts->{nbd}) {
5945         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5946
5947         my $migrate_storage_uri;
5948         # nbd_protocol_version > 0 for unix socket support
5949         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5950             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5951             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5952             $migrate_storage_uri = "nbd:unix:$socket_path";
5953             $res->{migrate}->{unix_sockets} = [$socket_path];
5954         } else {
5955             my $nodename = nodename();
5956             my $localip = $get_migration_ip->($nodename);
5957             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5958             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5959
5960             mon_cmd($vmid, "nbd-server-start", addr => {
5961                 type => 'inet',
5962                 data => {
5963                     host => "${localip}",
5964                     port => "${storage_migrate_port}",
5965                 },
5966             });
5967             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5968             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5969         }
5970
5971         my $block_info = mon_cmd($vmid, "query-block");
5972         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5973
5974         foreach my $opt (sort keys %$nbd) {
5975             my $drivestr = $nbd->{$opt}->{drivestr};
5976             my $volid = $nbd->{$opt}->{volid};
5977
5978             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5979
5980             mon_cmd(
5981                 $vmid,
5982                 "block-export-add",
5983                 id => "drive-$opt",
5984                 'node-name' => $block_node,
5985                 writable => JSON::true,
5986                 type => "nbd",
5987                 name => "drive-$opt", # NBD export name
5988             );
5989
5990             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5991             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5992             print "re-using replicated volume: $opt - $volid\n"
5993                 if $nbd->{$opt}->{replicated};
5994
5995             $res->{drives}->{$opt} = $nbd->{$opt};
5996             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5997         }
5998     }
5999
6000     if ($migratedfrom) {
6001         eval {
6002             set_migration_caps($vmid);
6003         };
6004         warn $@ if $@;
6005
6006         if ($spice_port) {
6007             print "spice listens on port $spice_port\n";
6008             $res->{spice_port} = $spice_port;
6009             if ($migrate_opts->{spice_ticket}) {
6010                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6011                     $migrate_opts->{spice_ticket});
6012                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6013             }
6014         }
6015
6016     } else {
6017         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6018             if !$statefile && $conf->{balloon};
6019
6020         foreach my $opt (keys %$conf) {
6021             next if $opt !~  m/^net\d+$/;
6022             my $nicconf = parse_net($conf->{$opt});
6023             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6024         }
6025         add_nets_bridge_fdb($conf, $vmid);
6026     }
6027
6028    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6029         eval {
6030             mon_cmd(
6031                 $vmid,
6032                 'qom-set',
6033                 path => "machine/peripheral/balloon0",
6034                 property => "guest-stats-polling-interval",
6035                 value => 2
6036             );
6037         };
6038         log_warn("could not set polling interval for ballooning - $@") if $@;
6039     }
6040
6041     if ($resume) {
6042         print "Resumed VM, removing state\n";
6043         if (my $vmstate = $conf->{vmstate}) {
6044             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6045             PVE::Storage::vdisk_free($storecfg, $vmstate);
6046         }
6047         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6048         PVE::QemuConfig->write_config($vmid, $conf);
6049     }
6050
6051     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6052
6053     my ($current_machine, $is_deprecated) =
6054         PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6055     if ($is_deprecated) {
6056         log_warn(
6057             "current machine version '$current_machine' is deprecated - see the documentation and ".
6058             "change to a newer one",
6059         );
6060     }
6061
6062     return $res;
6063 }
6064
6065 sub vm_commandline {
6066     my ($storecfg, $vmid, $snapname) = @_;
6067
6068     my $conf = PVE::QemuConfig->load_config($vmid);
6069
6070     my ($forcemachine, $forcecpu);
6071     if ($snapname) {
6072         my $snapshot = $conf->{snapshots}->{$snapname};
6073         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6074
6075         # check for machine or CPU overrides in snapshot
6076         $forcemachine = $snapshot->{runningmachine};
6077         $forcecpu = $snapshot->{runningcpu};
6078
6079         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6080
6081         $conf = $snapshot;
6082     }
6083
6084     my $defaults = load_defaults();
6085
6086     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6087
6088     return PVE::Tools::cmd2string($cmd);
6089 }
6090
6091 sub vm_reset {
6092     my ($vmid, $skiplock) = @_;
6093
6094     PVE::QemuConfig->lock_config($vmid, sub {
6095
6096         my $conf = PVE::QemuConfig->load_config($vmid);
6097
6098         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6099
6100         mon_cmd($vmid, "system_reset");
6101     });
6102 }
6103
6104 sub get_vm_volumes {
6105     my ($conf) = @_;
6106
6107     my $vollist = [];
6108     foreach_volid($conf, sub {
6109         my ($volid, $attr) = @_;
6110
6111         return if $volid =~ m|^/|;
6112
6113         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6114         return if !$sid;
6115
6116         push @$vollist, $volid;
6117     });
6118
6119     return $vollist;
6120 }
6121
6122 sub cleanup_pci_devices {
6123     my ($vmid, $conf) = @_;
6124
6125     foreach my $key (keys %$conf) {
6126         next if $key !~ m/^hostpci(\d+)$/;
6127         my $hostpciindex = $1;
6128         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6129         my $d = parse_hostpci($conf->{$key});
6130         if ($d->{mdev}) {
6131             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6132             # don't want to break ABI just for this two liner
6133             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6134
6135             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6136             # out when we do it first. so wait for up to 10 seconds and then try it manually
6137             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/ && -e $dev_sysfs_dir) {
6138                 my $count = 0;
6139                 while (-e $dev_sysfs_dir && $count < 10) {
6140                     sleep 1;
6141                     $count++;
6142                 }
6143                 print "waited $count seconds for mediated device driver finishing clean up\n";
6144             }
6145
6146             if (-e $dev_sysfs_dir) {
6147                 print "actively clean up mediated device with UUID $uuid\n";
6148                 PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1");
6149             }
6150         }
6151     }
6152     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6153 }
6154
6155 sub vm_stop_cleanup {
6156     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6157
6158     eval {
6159
6160         if (!$keepActive) {
6161             my $vollist = get_vm_volumes($conf);
6162             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6163
6164             if (my $tpmdrive = $conf->{tpmstate0}) {
6165                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6166                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6167                 if ($storeid) {
6168                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6169                 }
6170             }
6171         }
6172
6173         foreach my $ext (qw(mon qmp pid vnc qga)) {
6174             unlink "/var/run/qemu-server/${vmid}.$ext";
6175         }
6176
6177         if ($conf->{ivshmem}) {
6178             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6179             # just delete it for now, VMs which have this already open do not
6180             # are affected, but new VMs will get a separated one. If this
6181             # becomes an issue we either add some sort of ref-counting or just
6182             # add a "don't delete on stop" flag to the ivshmem format.
6183             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6184         }
6185
6186         cleanup_pci_devices($vmid, $conf);
6187
6188         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6189     };
6190     warn $@ if $@; # avoid errors - just warn
6191 }
6192
6193 # call only in locked context
6194 sub _do_vm_stop {
6195     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6196
6197     my $pid = check_running($vmid, $nocheck);
6198     return if !$pid;
6199
6200     my $conf;
6201     if (!$nocheck) {
6202         $conf = PVE::QemuConfig->load_config($vmid);
6203         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6204         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6205             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6206             $timeout = $opts->{down} if $opts->{down};
6207         }
6208         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6209     }
6210
6211     eval {
6212         if ($shutdown) {
6213             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6214                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6215             } else {
6216                 mon_cmd($vmid, "system_powerdown");
6217             }
6218         } else {
6219             mon_cmd($vmid, "quit");
6220         }
6221     };
6222     my $err = $@;
6223
6224     if (!$err) {
6225         $timeout = 60 if !defined($timeout);
6226
6227         my $count = 0;
6228         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6229             $count++;
6230             sleep 1;
6231         }
6232
6233         if ($count >= $timeout) {
6234             if ($force) {
6235                 warn "VM still running - terminating now with SIGTERM\n";
6236                 kill 15, $pid;
6237             } else {
6238                 die "VM quit/powerdown failed - got timeout\n";
6239             }
6240         } else {
6241             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6242             return;
6243         }
6244     } else {
6245        if (!check_running($vmid, $nocheck)) {
6246             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6247             return;
6248        }
6249        if ($force) {
6250             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6251             kill 15, $pid;
6252         } else {
6253             die "VM quit/powerdown failed\n";
6254         }
6255     }
6256
6257     # wait again
6258     $timeout = 10;
6259
6260     my $count = 0;
6261     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6262         $count++;
6263         sleep 1;
6264     }
6265
6266     if ($count >= $timeout) {
6267         warn "VM still running - terminating now with SIGKILL\n";
6268         kill 9, $pid;
6269         sleep 1;
6270     }
6271
6272     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6273 }
6274
6275 # Note: use $nocheck to skip tests if VM configuration file exists.
6276 # We need that when migration VMs to other nodes (files already moved)
6277 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6278 sub vm_stop {
6279     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6280
6281     $force = 1 if !defined($force) && !$shutdown;
6282
6283     if ($migratedfrom){
6284         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6285         kill 15, $pid if $pid;
6286         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6287         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6288         return;
6289     }
6290
6291     PVE::QemuConfig->lock_config($vmid, sub {
6292         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6293    });
6294 }
6295
6296 sub vm_reboot {
6297     my ($vmid, $timeout) = @_;
6298
6299     PVE::QemuConfig->lock_config($vmid, sub {
6300         eval {
6301
6302             # only reboot if running, as qmeventd starts it again on a stop event
6303             return if !check_running($vmid);
6304
6305             create_reboot_request($vmid);
6306
6307             my $storecfg = PVE::Storage::config();
6308             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6309
6310         };
6311         if (my $err = $@) {
6312             # avoid that the next normal shutdown will be confused for a reboot
6313             clear_reboot_request($vmid);
6314             die $err;
6315         }
6316    });
6317 }
6318
6319 # note: if using the statestorage parameter, the caller has to check privileges
6320 sub vm_suspend {
6321     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6322
6323     my $conf;
6324     my $path;
6325     my $storecfg;
6326     my $vmstate;
6327
6328     PVE::QemuConfig->lock_config($vmid, sub {
6329
6330         $conf = PVE::QemuConfig->load_config($vmid);
6331
6332         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6333         PVE::QemuConfig->check_lock($conf)
6334             if !($skiplock || $is_backing_up);
6335
6336         die "cannot suspend to disk during backup\n"
6337             if $is_backing_up && $includestate;
6338
6339         if ($includestate) {
6340             $conf->{lock} = 'suspending';
6341             my $date = strftime("%Y-%m-%d", localtime(time()));
6342             $storecfg = PVE::Storage::config();
6343             if (!$statestorage) {
6344                 $statestorage = find_vmstate_storage($conf, $storecfg);
6345                 # check permissions for the storage
6346                 my $rpcenv = PVE::RPCEnvironment::get();
6347                 if ($rpcenv->{type} ne 'cli') {
6348                     my $authuser = $rpcenv->get_user();
6349                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6350                 }
6351             }
6352
6353
6354             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6355                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6356             $path = PVE::Storage::path($storecfg, $vmstate);
6357             PVE::QemuConfig->write_config($vmid, $conf);
6358         } else {
6359             mon_cmd($vmid, "stop");
6360         }
6361     });
6362
6363     if ($includestate) {
6364         # save vm state
6365         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6366
6367         eval {
6368             set_migration_caps($vmid, 1);
6369             mon_cmd($vmid, "savevm-start", statefile => $path);
6370             for(;;) {
6371                 my $state = mon_cmd($vmid, "query-savevm");
6372                 if (!$state->{status}) {
6373                     die "savevm not active\n";
6374                 } elsif ($state->{status} eq 'active') {
6375                     sleep(1);
6376                     next;
6377                 } elsif ($state->{status} eq 'completed') {
6378                     print "State saved, quitting\n";
6379                     last;
6380                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6381                     die "query-savevm failed with error '$state->{error}'\n"
6382                 } else {
6383                     die "query-savevm returned status '$state->{status}'\n";
6384                 }
6385             }
6386         };
6387         my $err = $@;
6388
6389         PVE::QemuConfig->lock_config($vmid, sub {
6390             $conf = PVE::QemuConfig->load_config($vmid);
6391             if ($err) {
6392                 # cleanup, but leave suspending lock, to indicate something went wrong
6393                 eval {
6394                     mon_cmd($vmid, "savevm-end");
6395                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6396                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6397                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6398                     PVE::QemuConfig->write_config($vmid, $conf);
6399                 };
6400                 warn $@ if $@;
6401                 die $err;
6402             }
6403
6404             die "lock changed unexpectedly\n"
6405                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6406
6407             mon_cmd($vmid, "quit");
6408             $conf->{lock} = 'suspended';
6409             PVE::QemuConfig->write_config($vmid, $conf);
6410         });
6411     }
6412 }
6413
6414 # $nocheck is set when called as part of a migration - in this context the
6415 # location of the config file (source or target node) is not deterministic,
6416 # since migration cannot wait for pmxcfs to process the rename
6417 sub vm_resume {
6418     my ($vmid, $skiplock, $nocheck) = @_;
6419
6420     PVE::QemuConfig->lock_config($vmid, sub {
6421         my $res = mon_cmd($vmid, 'query-status');
6422         my $resume_cmd = 'cont';
6423         my $reset = 0;
6424         my $conf;
6425         if ($nocheck) {
6426             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6427             if ($@) {
6428                 my $vmlist = PVE::Cluster::get_vmlist();
6429                 if (exists($vmlist->{ids}->{$vmid})) {
6430                     my $node = $vmlist->{ids}->{$vmid}->{node};
6431                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6432                 }
6433                 if (!$conf) {
6434                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6435                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6436                 }
6437             }
6438         } else {
6439             $conf = PVE::QemuConfig->load_config($vmid);
6440         }
6441
6442         if ($res->{status}) {
6443             return if $res->{status} eq 'running'; # job done, go home
6444             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6445             $reset = 1 if $res->{status} eq 'shutdown';
6446         }
6447
6448         if (!$nocheck) {
6449             PVE::QemuConfig->check_lock($conf)
6450                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6451         }
6452
6453         if ($reset) {
6454             # required if a VM shuts down during a backup and we get a resume
6455             # request before the backup finishes for example
6456             mon_cmd($vmid, "system_reset");
6457         }
6458
6459         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6460
6461         mon_cmd($vmid, $resume_cmd);
6462     });
6463 }
6464
6465 sub vm_sendkey {
6466     my ($vmid, $skiplock, $key) = @_;
6467
6468     PVE::QemuConfig->lock_config($vmid, sub {
6469
6470         my $conf = PVE::QemuConfig->load_config($vmid);
6471
6472         # there is no qmp command, so we use the human monitor command
6473         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6474         die $res if $res ne '';
6475     });
6476 }
6477
6478 sub check_bridge_access {
6479     my ($rpcenv, $authuser, $conf) = @_;
6480
6481     return 1 if $authuser eq 'root@pam';
6482
6483     for my $opt (sort keys $conf->%*) {
6484         next if $opt !~ m/^net\d+$/;
6485         my $net = parse_net($conf->{$opt});
6486         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6487         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6488     }
6489     return 1;
6490 };
6491
6492 sub check_mapping_access {
6493     my ($rpcenv, $user, $conf) = @_;
6494
6495     for my $opt (keys $conf->%*) {
6496         if ($opt =~ m/^usb\d+$/) {
6497             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6498             if (my $host = $device->{host}) {
6499                 die "only root can set '$opt' config for real devices\n"
6500                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6501             } elsif ($device->{mapping}) {
6502                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6503             } else {
6504                 die "either 'host' or 'mapping' must be set.\n";
6505             }
6506         } elsif ($opt =~ m/^hostpci\d+$/) {
6507             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6508             if ($device->{host}) {
6509                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6510             } elsif ($device->{mapping}) {
6511                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6512             } else {
6513                 die "either 'host' or 'mapping' must be set.\n";
6514             }
6515        }
6516    }
6517 };
6518
6519 sub check_restore_permissions {
6520     my ($rpcenv, $user, $conf) = @_;
6521
6522     check_bridge_access($rpcenv, $user, $conf);
6523     check_mapping_access($rpcenv, $user, $conf);
6524 }
6525 # vzdump restore implementaion
6526
6527 sub tar_archive_read_firstfile {
6528     my $archive = shift;
6529
6530     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6531
6532     # try to detect archive type first
6533     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6534         die "unable to open file '$archive'\n";
6535     my $firstfile = <$fh>;
6536     kill 15, $pid;
6537     close $fh;
6538
6539     die "ERROR: archive contaions no data\n" if !$firstfile;
6540     chomp $firstfile;
6541
6542     return $firstfile;
6543 }
6544
6545 sub tar_restore_cleanup {
6546     my ($storecfg, $statfile) = @_;
6547
6548     print STDERR "starting cleanup\n";
6549
6550     if (my $fd = IO::File->new($statfile, "r")) {
6551         while (defined(my $line = <$fd>)) {
6552             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6553                 my $volid = $2;
6554                 eval {
6555                     if ($volid =~ m|^/|) {
6556                         unlink $volid || die 'unlink failed\n';
6557                     } else {
6558                         PVE::Storage::vdisk_free($storecfg, $volid);
6559                     }
6560                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6561                 };
6562                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6563             } else {
6564                 print STDERR "unable to parse line in statfile - $line";
6565             }
6566         }
6567         $fd->close();
6568     }
6569 }
6570
6571 sub restore_file_archive {
6572     my ($archive, $vmid, $user, $opts) = @_;
6573
6574     return restore_vma_archive($archive, $vmid, $user, $opts)
6575         if $archive eq '-';
6576
6577     my $info = PVE::Storage::archive_info($archive);
6578     my $format = $opts->{format} // $info->{format};
6579     my $comp = $info->{compression};
6580
6581     # try to detect archive format
6582     if ($format eq 'tar') {
6583         return restore_tar_archive($archive, $vmid, $user, $opts);
6584     } else {
6585         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6586     }
6587 }
6588
6589 # hepler to remove disks that will not be used after restore
6590 my $restore_cleanup_oldconf = sub {
6591     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6592
6593     my $kept_disks = {};
6594
6595     PVE::QemuConfig->foreach_volume($oldconf, sub {
6596         my ($ds, $drive) = @_;
6597
6598         return if drive_is_cdrom($drive, 1);
6599
6600         my $volid = $drive->{file};
6601         return if !$volid || $volid =~ m|^/|;
6602
6603         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6604         return if !$path || !$owner || ($owner != $vmid);
6605
6606         # Note: only delete disk we want to restore
6607         # other volumes will become unused
6608         if ($virtdev_hash->{$ds}) {
6609             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6610             if (my $err = $@) {
6611                 warn $err;
6612             }
6613         } else {
6614             $kept_disks->{$volid} = 1;
6615         }
6616     });
6617
6618     # after the restore we have no snapshots anymore
6619     for my $snapname (keys $oldconf->{snapshots}->%*) {
6620         my $snap = $oldconf->{snapshots}->{$snapname};
6621         if ($snap->{vmstate}) {
6622             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6623             if (my $err = $@) {
6624                 warn $err;
6625             }
6626         }
6627
6628         for my $volid (keys $kept_disks->%*) {
6629             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6630             warn $@ if $@;
6631         }
6632     }
6633 };
6634
6635 # Helper to parse vzdump backup device hints
6636 #
6637 # $rpcenv: Environment, used to ckeck storage permissions
6638 # $user: User ID, to check storage permissions
6639 # $storecfg: Storage configuration
6640 # $fh: the file handle for reading the configuration
6641 # $devinfo: should contain device sizes for all backu-up'ed devices
6642 # $options: backup options (pool, default storage)
6643 #
6644 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6645 my $parse_backup_hints = sub {
6646     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6647
6648     my $check_storage = sub { # assert if an image can be allocate
6649         my ($storeid, $scfg) = @_;
6650         die "Content type 'images' is not available on storage '$storeid'\n"
6651             if !$scfg->{content}->{images};
6652         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6653             if $user ne 'root@pam';
6654     };
6655
6656     my $virtdev_hash = {};
6657     while (defined(my $line = <$fh>)) {
6658         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6659             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6660             die "archive does not contain data for drive '$virtdev'\n"
6661                 if !$devinfo->{$devname};
6662
6663             if (defined($options->{storage})) {
6664                 $storeid = $options->{storage} || 'local';
6665             } elsif (!$storeid) {
6666                 $storeid = 'local';
6667             }
6668             $format = 'raw' if !$format;
6669             $devinfo->{$devname}->{devname} = $devname;
6670             $devinfo->{$devname}->{virtdev} = $virtdev;
6671             $devinfo->{$devname}->{format} = $format;
6672             $devinfo->{$devname}->{storeid} = $storeid;
6673
6674             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6675             $check_storage->($storeid, $scfg); # permission and content type check
6676
6677             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6678         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6679             my $virtdev = $1;
6680             my $drive = parse_drive($virtdev, $2);
6681
6682             if (drive_is_cloudinit($drive)) {
6683                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6684                 $storeid = $options->{storage} if defined ($options->{storage});
6685                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6686                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6687
6688                 $check_storage->($storeid, $scfg); # permission and content type check
6689
6690                 $virtdev_hash->{$virtdev} = {
6691                     format => $format,
6692                     storeid => $storeid,
6693                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6694                     is_cloudinit => 1,
6695                 };
6696             }
6697         }
6698     }
6699
6700     return $virtdev_hash;
6701 };
6702
6703 # Helper to allocate and activate all volumes required for a restore
6704 #
6705 # $storecfg: Storage configuration
6706 # $virtdev_hash: as returned by parse_backup_hints()
6707 #
6708 # Returns: { $virtdev => $volid }
6709 my $restore_allocate_devices = sub {
6710     my ($storecfg, $virtdev_hash, $vmid) = @_;
6711
6712     my $map = {};
6713     foreach my $virtdev (sort keys %$virtdev_hash) {
6714         my $d = $virtdev_hash->{$virtdev};
6715         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6716         my $storeid = $d->{storeid};
6717         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6718
6719         # test if requested format is supported
6720         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6721         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6722         $d->{format} = $defFormat if !$supported;
6723
6724         my $name;
6725         if ($d->{is_cloudinit}) {
6726             $name = "vm-$vmid-cloudinit";
6727             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6728             if ($scfg->{path}) {
6729                 $name .= ".$d->{format}";
6730             }
6731         }
6732
6733         my $volid = PVE::Storage::vdisk_alloc(
6734             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6735
6736         print STDERR "new volume ID is '$volid'\n";
6737         $d->{volid} = $volid;
6738
6739         PVE::Storage::activate_volumes($storecfg, [$volid]);
6740
6741         $map->{$virtdev} = $volid;
6742     }
6743
6744     return $map;
6745 };
6746
6747 sub restore_update_config_line {
6748     my ($cookie, $map, $line, $unique) = @_;
6749
6750     return '' if $line =~ m/^\#qmdump\#/;
6751     return '' if $line =~ m/^\#vzdump\#/;
6752     return '' if $line =~ m/^lock:/;
6753     return '' if $line =~ m/^unused\d+:/;
6754     return '' if $line =~ m/^parent:/;
6755
6756     my $res = '';
6757
6758     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6759     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6760         # try to convert old 1.X settings
6761         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6762         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6763             my ($model, $macaddr) = split(/\=/, $devconfig);
6764             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6765             my $net = {
6766                 model => $model,
6767                 bridge => "vmbr$ind",
6768                 macaddr => $macaddr,
6769             };
6770             my $netstr = print_net($net);
6771
6772             $res .= "net$cookie->{netcount}: $netstr\n";
6773             $cookie->{netcount}++;
6774         }
6775     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6776         my ($id, $netstr) = ($1, $2);
6777         my $net = parse_net($netstr);
6778         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6779         $netstr = print_net($net);
6780         $res .= "$id: $netstr\n";
6781     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6782         my $virtdev = $1;
6783         my $value = $3;
6784         my $di = parse_drive($virtdev, $value);
6785         if (defined($di->{backup}) && !$di->{backup}) {
6786             $res .= "#$line";
6787         } elsif ($map->{$virtdev}) {
6788             delete $di->{format}; # format can change on restore
6789             $di->{file} = $map->{$virtdev};
6790             $value = print_drive($di);
6791             $res .= "$virtdev: $value\n";
6792         } else {
6793             $res .= $line;
6794         }
6795     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6796         my $vmgenid = $1;
6797         if ($vmgenid ne '0') {
6798             # always generate a new vmgenid if there was a valid one setup
6799             $vmgenid = generate_uuid();
6800         }
6801         $res .= "vmgenid: $vmgenid\n";
6802     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6803         my ($uuid, $uuid_str);
6804         UUID::generate($uuid);
6805         UUID::unparse($uuid, $uuid_str);
6806         my $smbios1 = parse_smbios1($2);
6807         $smbios1->{uuid} = $uuid_str;
6808         $res .= $1.print_smbios1($smbios1)."\n";
6809     } else {
6810         $res .= $line;
6811     }
6812
6813     return $res;
6814 }
6815
6816 my $restore_deactivate_volumes = sub {
6817     my ($storecfg, $virtdev_hash) = @_;
6818
6819     my $vollist = [];
6820     for my $dev (values $virtdev_hash->%*) {
6821         push $vollist->@*, $dev->{volid} if $dev->{volid};
6822     }
6823
6824     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6825     print STDERR $@ if $@;
6826 };
6827
6828 my $restore_destroy_volumes = sub {
6829     my ($storecfg, $virtdev_hash) = @_;
6830
6831     for my $dev (values $virtdev_hash->%*) {
6832         my $volid = $dev->{volid} or next;
6833         eval {
6834             PVE::Storage::vdisk_free($storecfg, $volid);
6835             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6836         };
6837         print STDERR "unable to cleanup '$volid' - $@" if $@;
6838     }
6839 };
6840
6841 sub restore_merge_config {
6842     my ($filename, $backup_conf_raw, $override_conf) = @_;
6843
6844     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6845     for my $key (keys $override_conf->%*) {
6846         $backup_conf->{$key} = $override_conf->{$key};
6847     }
6848
6849     return $backup_conf;
6850 }
6851
6852 sub scan_volids {
6853     my ($cfg, $vmid) = @_;
6854
6855     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6856
6857     my $volid_hash = {};
6858     foreach my $storeid (keys %$info) {
6859         foreach my $item (@{$info->{$storeid}}) {
6860             next if !($item->{volid} && $item->{size});
6861             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6862             $volid_hash->{$item->{volid}} = $item;
6863         }
6864     }
6865
6866     return $volid_hash;
6867 }
6868
6869 sub update_disk_config {
6870     my ($vmid, $conf, $volid_hash) = @_;
6871
6872     my $changes;
6873     my $prefix = "VM $vmid";
6874
6875     # used and unused disks
6876     my $referenced = {};
6877
6878     # Note: it is allowed to define multiple storages with same path (alias), so
6879     # we need to check both 'volid' and real 'path' (two different volid can point
6880     # to the same path).
6881
6882     my $referencedpath = {};
6883
6884     # update size info
6885     PVE::QemuConfig->foreach_volume($conf, sub {
6886         my ($opt, $drive) = @_;
6887
6888         my $volid = $drive->{file};
6889         return if !$volid;
6890         my $volume = $volid_hash->{$volid};
6891
6892         # mark volid as "in-use" for next step
6893         $referenced->{$volid} = 1;
6894         if ($volume && (my $path = $volume->{path})) {
6895             $referencedpath->{$path} = 1;
6896         }
6897
6898         return if drive_is_cdrom($drive);
6899         return if !$volume;
6900
6901         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6902         if (defined($updated)) {
6903             $changes = 1;
6904             $conf->{$opt} = print_drive($updated);
6905             print "$prefix ($opt): $msg\n";
6906         }
6907     });
6908
6909     # remove 'unusedX' entry if volume is used
6910     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6911         my ($opt, $drive) = @_;
6912
6913         my $volid = $drive->{file};
6914         return if !$volid;
6915
6916         my $path;
6917         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6918         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6919             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6920             $changes = 1;
6921             delete $conf->{$opt};
6922         }
6923
6924         $referenced->{$volid} = 1;
6925         $referencedpath->{$path} = 1 if $path;
6926     });
6927
6928     foreach my $volid (sort keys %$volid_hash) {
6929         next if $volid =~ m/vm-$vmid-state-/;
6930         next if $referenced->{$volid};
6931         my $path = $volid_hash->{$volid}->{path};
6932         next if !$path; # just to be sure
6933         next if $referencedpath->{$path};
6934         $changes = 1;
6935         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6936         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6937         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6938     }
6939
6940     return $changes;
6941 }
6942
6943 sub rescan {
6944     my ($vmid, $nolock, $dryrun) = @_;
6945
6946     my $cfg = PVE::Storage::config();
6947
6948     print "rescan volumes...\n";
6949     my $volid_hash = scan_volids($cfg, $vmid);
6950
6951     my $updatefn =  sub {
6952         my ($vmid) = @_;
6953
6954         my $conf = PVE::QemuConfig->load_config($vmid);
6955
6956         PVE::QemuConfig->check_lock($conf);
6957
6958         my $vm_volids = {};
6959         foreach my $volid (keys %$volid_hash) {
6960             my $info = $volid_hash->{$volid};
6961             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6962         }
6963
6964         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6965
6966         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6967     };
6968
6969     if (defined($vmid)) {
6970         if ($nolock) {
6971             &$updatefn($vmid);
6972         } else {
6973             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6974         }
6975     } else {
6976         my $vmlist = config_list();
6977         foreach my $vmid (keys %$vmlist) {
6978             if ($nolock) {
6979                 &$updatefn($vmid);
6980             } else {
6981                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6982             }
6983         }
6984     }
6985 }
6986
6987 sub restore_proxmox_backup_archive {
6988     my ($archive, $vmid, $user, $options) = @_;
6989
6990     my $storecfg = PVE::Storage::config();
6991
6992     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6993     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6994
6995     my $fingerprint = $scfg->{fingerprint};
6996     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6997
6998     my $repo = PVE::PBSClient::get_repository($scfg);
6999     my $namespace = $scfg->{namespace};
7000
7001     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7002     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7003     local $ENV{PBS_PASSWORD} = $password;
7004     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7005
7006     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7007         PVE::Storage::parse_volname($storecfg, $archive);
7008
7009     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7010
7011     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7012
7013     my $tmpdir = "/var/tmp/vzdumptmp$$";
7014     rmtree $tmpdir;
7015     mkpath $tmpdir;
7016
7017     my $conffile = PVE::QemuConfig->config_file($vmid);
7018      # disable interrupts (always do cleanups)
7019     local $SIG{INT} =
7020         local $SIG{TERM} =
7021         local $SIG{QUIT} =
7022         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7023
7024     # Note: $oldconf is undef if VM does not exists
7025     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7026     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7027     my $new_conf_raw = '';
7028
7029     my $rpcenv = PVE::RPCEnvironment::get();
7030     my $devinfo = {}; # info about drives included in backup
7031     my $virtdev_hash = {}; # info about allocated drives
7032
7033     eval {
7034         # enable interrupts
7035         local $SIG{INT} =
7036             local $SIG{TERM} =
7037             local $SIG{QUIT} =
7038             local $SIG{HUP} =
7039             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7040
7041         my $cfgfn = "$tmpdir/qemu-server.conf";
7042         my $firewall_config_fn = "$tmpdir/fw.conf";
7043         my $index_fn = "$tmpdir/index.json";
7044
7045         my $cmd = "restore";
7046
7047         my $param = [$pbs_backup_name, "index.json", $index_fn];
7048         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7049         my $index = PVE::Tools::file_get_contents($index_fn);
7050         $index = decode_json($index);
7051
7052         foreach my $info (@{$index->{files}}) {
7053             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7054                 my $devname = $1;
7055                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7056                     $devinfo->{$devname}->{size} = $1;
7057                 } else {
7058                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7059                 }
7060             }
7061         }
7062
7063         my $is_qemu_server_backup = scalar(
7064             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7065         );
7066         if (!$is_qemu_server_backup) {
7067             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7068         }
7069         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7070
7071         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7072         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7073
7074         if ($has_firewall_config) {
7075             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7076             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7077
7078             my $pve_firewall_dir = '/etc/pve/firewall';
7079             mkdir $pve_firewall_dir; # make sure the dir exists
7080             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7081         }
7082
7083         my $fh = IO::File->new($cfgfn, "r") ||
7084             die "unable to read qemu-server.conf - $!\n";
7085
7086         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7087
7088         # fixme: rate limit?
7089
7090         # create empty/temp config
7091         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7092
7093         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7094
7095         # allocate volumes
7096         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7097
7098         foreach my $virtdev (sort keys %$virtdev_hash) {
7099             my $d = $virtdev_hash->{$virtdev};
7100             next if $d->{is_cloudinit}; # no need to restore cloudinit
7101
7102             # this fails if storage is unavailable
7103             my $volid = $d->{volid};
7104             my $path = PVE::Storage::path($storecfg, $volid);
7105
7106             # for live-restore we only want to preload the efidisk and TPM state
7107             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7108
7109             my @ns_arg;
7110             if (defined(my $ns = $scfg->{namespace})) {
7111                 @ns_arg = ('--ns', $ns);
7112             }
7113
7114             my $pbs_restore_cmd = [
7115                 '/usr/bin/pbs-restore',
7116                 '--repository', $repo,
7117                 @ns_arg,
7118                 $pbs_backup_name,
7119                 "$d->{devname}.img.fidx",
7120                 $path,
7121                 '--verbose',
7122                 ];
7123
7124             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7125             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7126
7127             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7128                 push @$pbs_restore_cmd, '--skip-zero';
7129             }
7130
7131             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7132             print "restore proxmox backup image: $dbg_cmdstring\n";
7133             run_command($pbs_restore_cmd);
7134         }
7135
7136         $fh->seek(0, 0) || die "seek failed - $!\n";
7137
7138         my $cookie = { netcount => 0 };
7139         while (defined(my $line = <$fh>)) {
7140             $new_conf_raw .= restore_update_config_line(
7141                 $cookie,
7142                 $map,
7143                 $line,
7144                 $options->{unique},
7145             );
7146         }
7147
7148         $fh->close();
7149     };
7150     my $err = $@;
7151
7152     if ($err || !$options->{live}) {
7153         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7154     }
7155
7156     rmtree $tmpdir;
7157
7158     if ($err) {
7159         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7160         die $err;
7161     }
7162
7163     if ($options->{live}) {
7164         # keep lock during live-restore
7165         $new_conf_raw .= "\nlock: create";
7166     }
7167
7168     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7169     check_restore_permissions($rpcenv, $user, $new_conf);
7170     PVE::QemuConfig->write_config($vmid, $new_conf);
7171
7172     eval { rescan($vmid, 1); };
7173     warn $@ if $@;
7174
7175     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7176
7177     if ($options->{live}) {
7178         # enable interrupts
7179         local $SIG{INT} =
7180             local $SIG{TERM} =
7181             local $SIG{QUIT} =
7182             local $SIG{HUP} =
7183             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7184
7185         my $conf = PVE::QemuConfig->load_config($vmid);
7186         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7187
7188         # these special drives are already restored before start
7189         delete $devinfo->{'drive-efidisk0'};
7190         delete $devinfo->{'drive-tpmstate0-backup'};
7191
7192         my $pbs_opts = {
7193             repo => $repo,
7194             keyfile => $keyfile,
7195             snapshot => $pbs_backup_name,
7196             namespace => $namespace,
7197         };
7198         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7199
7200         PVE::QemuConfig->remove_lock($vmid, "create");
7201     }
7202 }
7203
7204 sub pbs_live_restore {
7205     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7206
7207     print "starting VM for live-restore\n";
7208     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7209
7210     my $pbs_backing = {};
7211     for my $ds (keys %$restored_disks) {
7212         $ds =~ m/^drive-(.*)$/;
7213         my $confname = $1;
7214         $pbs_backing->{$confname} = {
7215             repository => $opts->{repo},
7216             snapshot => $opts->{snapshot},
7217             archive => "$ds.img.fidx",
7218         };
7219         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7220         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7221
7222         my $drive = parse_drive($confname, $conf->{$confname});
7223         print "restoring '$ds' to '$drive->{file}'\n";
7224     }
7225
7226     my $drives_streamed = 0;
7227     eval {
7228         # make sure HA doesn't interrupt our restore by stopping the VM
7229         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7230             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7231         }
7232
7233         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7234         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7235         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7236
7237         my $qmeventd_fd = register_qmeventd_handle($vmid);
7238
7239         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7240         # this will effectively collapse the backing image chain consisting of
7241         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7242         # removes itself once all backing images vanish with 'auto-remove=on')
7243         my $jobs = {};
7244         for my $ds (sort keys %$restored_disks) {
7245             my $job_id = "restore-$ds";
7246             mon_cmd($vmid, 'block-stream',
7247                 'job-id' => $job_id,
7248                 device => "$ds",
7249             );
7250             $jobs->{$job_id} = {};
7251         }
7252
7253         mon_cmd($vmid, 'cont');
7254         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7255
7256         print "restore-drive jobs finished successfully, removing all tracking block devices"
7257             ." to disconnect from Proxmox Backup Server\n";
7258
7259         for my $ds (sort keys %$restored_disks) {
7260             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7261         }
7262
7263         close($qmeventd_fd);
7264     };
7265
7266     my $err = $@;
7267
7268     if ($err) {
7269         warn "An error occurred during live-restore: $err\n";
7270         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7271         die "live-restore failed\n";
7272     }
7273 }
7274
7275 sub restore_vma_archive {
7276     my ($archive, $vmid, $user, $opts, $comp) = @_;
7277
7278     my $readfrom = $archive;
7279
7280     my $cfg = PVE::Storage::config();
7281     my $commands = [];
7282     my $bwlimit = $opts->{bwlimit};
7283
7284     my $dbg_cmdstring = '';
7285     my $add_pipe = sub {
7286         my ($cmd) = @_;
7287         push @$commands, $cmd;
7288         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7289         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7290         $readfrom = '-';
7291     };
7292
7293     my $input = undef;
7294     if ($archive eq '-') {
7295         $input = '<&STDIN';
7296     } else {
7297         # If we use a backup from a PVE defined storage we also consider that
7298         # storage's rate limit:
7299         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7300         if (defined($volid)) {
7301             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7302             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7303             if ($readlimit) {
7304                 print STDERR "applying read rate limit: $readlimit\n";
7305                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7306                 $add_pipe->($cstream);
7307             }
7308         }
7309     }
7310
7311     if ($comp) {
7312         my $info = PVE::Storage::decompressor_info('vma', $comp);
7313         my $cmd = $info->{decompressor};
7314         push @$cmd, $readfrom;
7315         $add_pipe->($cmd);
7316     }
7317
7318     my $tmpdir = "/var/tmp/vzdumptmp$$";
7319     rmtree $tmpdir;
7320
7321     # disable interrupts (always do cleanups)
7322     local $SIG{INT} =
7323         local $SIG{TERM} =
7324         local $SIG{QUIT} =
7325         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7326
7327     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7328     POSIX::mkfifo($mapfifo, 0600);
7329     my $fifofh;
7330     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7331
7332     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7333
7334     my $devinfo = {}; # info about drives included in backup
7335     my $virtdev_hash = {}; # info about allocated drives
7336
7337     my $rpcenv = PVE::RPCEnvironment::get();
7338
7339     my $conffile = PVE::QemuConfig->config_file($vmid);
7340
7341     # Note: $oldconf is undef if VM does not exist
7342     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7343     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7344     my $new_conf_raw = '';
7345
7346     my %storage_limits;
7347
7348     my $print_devmap = sub {
7349         my $cfgfn = "$tmpdir/qemu-server.conf";
7350
7351         # we can read the config - that is already extracted
7352         my $fh = IO::File->new($cfgfn, "r") ||
7353             die "unable to read qemu-server.conf - $!\n";
7354
7355         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7356         if (-f $fwcfgfn) {
7357             my $pve_firewall_dir = '/etc/pve/firewall';
7358             mkdir $pve_firewall_dir; # make sure the dir exists
7359             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7360         }
7361
7362         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7363
7364         foreach my $info (values %{$virtdev_hash}) {
7365             my $storeid = $info->{storeid};
7366             next if defined($storage_limits{$storeid});
7367
7368             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7369             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7370             $storage_limits{$storeid} = $limit * 1024;
7371         }
7372
7373         foreach my $devname (keys %$devinfo) {
7374             die "found no device mapping information for device '$devname'\n"
7375                 if !$devinfo->{$devname}->{virtdev};
7376         }
7377
7378         # create empty/temp config
7379         if ($oldconf) {
7380             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7381             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7382         }
7383
7384         # allocate volumes
7385         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7386
7387         # print restore information to $fifofh
7388         foreach my $virtdev (sort keys %$virtdev_hash) {
7389             my $d = $virtdev_hash->{$virtdev};
7390             next if $d->{is_cloudinit}; # no need to restore cloudinit
7391
7392             my $storeid = $d->{storeid};
7393             my $volid = $d->{volid};
7394
7395             my $map_opts = '';
7396             if (my $limit = $storage_limits{$storeid}) {
7397                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7398             }
7399
7400             my $write_zeros = 1;
7401             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7402                 $write_zeros = 0;
7403             }
7404
7405             my $path = PVE::Storage::path($cfg, $volid);
7406
7407             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7408
7409             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7410         }
7411
7412         $fh->seek(0, 0) || die "seek failed - $!\n";
7413
7414         my $cookie = { netcount => 0 };
7415         while (defined(my $line = <$fh>)) {
7416             $new_conf_raw .= restore_update_config_line(
7417                 $cookie,
7418                 $map,
7419                 $line,
7420                 $opts->{unique},
7421             );
7422         }
7423
7424         $fh->close();
7425     };
7426
7427     my $oldtimeout;
7428
7429     eval {
7430         # enable interrupts
7431         local $SIG{INT} =
7432             local $SIG{TERM} =
7433             local $SIG{QUIT} =
7434             local $SIG{HUP} =
7435             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7436         local $SIG{ALRM} = sub { die "got timeout\n"; };
7437
7438         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7439
7440         my $parser = sub {
7441             my $line = shift;
7442
7443             print "$line\n";
7444
7445             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7446                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7447                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7448             } elsif ($line =~ m/^CTIME: /) {
7449                 # we correctly received the vma config, so we can disable
7450                 # the timeout now for disk allocation
7451                 alarm($oldtimeout || 0);
7452                 $oldtimeout = undef;
7453                 &$print_devmap();
7454                 print $fifofh "done\n";
7455                 close($fifofh);
7456                 $fifofh = undef;
7457             }
7458         };
7459
7460         print "restore vma archive: $dbg_cmdstring\n";
7461         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7462     };
7463     my $err = $@;
7464
7465     alarm($oldtimeout) if $oldtimeout;
7466
7467     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7468
7469     close($fifofh) if $fifofh;
7470     unlink $mapfifo;
7471     rmtree $tmpdir;
7472
7473     if ($err) {
7474         $restore_destroy_volumes->($cfg, $virtdev_hash);
7475         die $err;
7476     }
7477
7478     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7479     check_restore_permissions($rpcenv, $user, $new_conf);
7480     PVE::QemuConfig->write_config($vmid, $new_conf);
7481
7482     eval { rescan($vmid, 1); };
7483     warn $@ if $@;
7484
7485     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7486 }
7487
7488 sub restore_tar_archive {
7489     my ($archive, $vmid, $user, $opts) = @_;
7490
7491     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7492         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7493         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7494     }
7495
7496     if ($archive ne '-') {
7497         my $firstfile = tar_archive_read_firstfile($archive);
7498         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7499             if $firstfile ne 'qemu-server.conf';
7500     }
7501
7502     my $storecfg = PVE::Storage::config();
7503
7504     # avoid zombie disks when restoring over an existing VM -> cleanup first
7505     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7506     # skiplock=1 because qmrestore has set the 'create' lock itself already
7507     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7508     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7509
7510     my $tocmd = "/usr/lib/qemu-server/qmextract";
7511
7512     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7513     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7514     $tocmd .= ' --prealloc' if $opts->{prealloc};
7515     $tocmd .= ' --info' if $opts->{info};
7516
7517     # tar option "xf" does not autodetect compression when read from STDIN,
7518     # so we pipe to zcat
7519     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7520         PVE::Tools::shellquote("--to-command=$tocmd");
7521
7522     my $tmpdir = "/var/tmp/vzdumptmp$$";
7523     mkpath $tmpdir;
7524
7525     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7526     local $ENV{VZDUMP_VMID} = $vmid;
7527     local $ENV{VZDUMP_USER} = $user;
7528
7529     my $conffile = PVE::QemuConfig->config_file($vmid);
7530     my $new_conf_raw = '';
7531
7532     # disable interrupts (always do cleanups)
7533     local $SIG{INT} =
7534         local $SIG{TERM} =
7535         local $SIG{QUIT} =
7536         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7537
7538     eval {
7539         # enable interrupts
7540         local $SIG{INT} =
7541             local $SIG{TERM} =
7542             local $SIG{QUIT} =
7543             local $SIG{HUP} =
7544             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7545
7546         if ($archive eq '-') {
7547             print "extracting archive from STDIN\n";
7548             run_command($cmd, input => "<&STDIN");
7549         } else {
7550             print "extracting archive '$archive'\n";
7551             run_command($cmd);
7552         }
7553
7554         return if $opts->{info};
7555
7556         # read new mapping
7557         my $map = {};
7558         my $statfile = "$tmpdir/qmrestore.stat";
7559         if (my $fd = IO::File->new($statfile, "r")) {
7560             while (defined (my $line = <$fd>)) {
7561                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7562                     $map->{$1} = $2 if $1;
7563                 } else {
7564                     print STDERR "unable to parse line in statfile - $line\n";
7565                 }
7566             }
7567             $fd->close();
7568         }
7569
7570         my $confsrc = "$tmpdir/qemu-server.conf";
7571
7572         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7573
7574         my $cookie = { netcount => 0 };
7575         while (defined (my $line = <$srcfd>)) {
7576             $new_conf_raw .= restore_update_config_line(
7577                 $cookie,
7578                 $map,
7579                 $line,
7580                 $opts->{unique},
7581             );
7582         }
7583
7584         $srcfd->close();
7585     };
7586     if (my $err = $@) {
7587         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7588         die $err;
7589     }
7590
7591     rmtree $tmpdir;
7592
7593     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7594
7595     PVE::Cluster::cfs_update(); # make sure we read new file
7596
7597     eval { rescan($vmid, 1); };
7598     warn $@ if $@;
7599 };
7600
7601 sub foreach_storage_used_by_vm {
7602     my ($conf, $func) = @_;
7603
7604     my $sidhash = {};
7605
7606     PVE::QemuConfig->foreach_volume($conf, sub {
7607         my ($ds, $drive) = @_;
7608         return if drive_is_cdrom($drive);
7609
7610         my $volid = $drive->{file};
7611
7612         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7613         $sidhash->{$sid} = $sid if $sid;
7614     });
7615
7616     foreach my $sid (sort keys %$sidhash) {
7617         &$func($sid);
7618     }
7619 }
7620
7621 my $qemu_snap_storage = {
7622     rbd => 1,
7623 };
7624 sub do_snapshots_with_qemu {
7625     my ($storecfg, $volid, $deviceid) = @_;
7626
7627     return if $deviceid =~ m/tpmstate0/;
7628
7629     my $storage_name = PVE::Storage::parse_volume_id($volid);
7630     my $scfg = $storecfg->{ids}->{$storage_name};
7631     die "could not find storage '$storage_name'\n" if !defined($scfg);
7632
7633     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7634         return 1;
7635     }
7636
7637     if ($volid =~ m/\.(qcow2|qed)$/){
7638         return 1;
7639     }
7640
7641     return;
7642 }
7643
7644 sub qga_check_running {
7645     my ($vmid, $nowarn) = @_;
7646
7647     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7648     if ($@) {
7649         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7650         return 0;
7651     }
7652     return 1;
7653 }
7654
7655 sub template_create {
7656     my ($vmid, $conf, $disk) = @_;
7657
7658     my $storecfg = PVE::Storage::config();
7659
7660     PVE::QemuConfig->foreach_volume($conf, sub {
7661         my ($ds, $drive) = @_;
7662
7663         return if drive_is_cdrom($drive);
7664         return if $disk && $ds ne $disk;
7665
7666         my $volid = $drive->{file};
7667         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7668
7669         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7670         $drive->{file} = $voliddst;
7671         $conf->{$ds} = print_drive($drive);
7672         PVE::QemuConfig->write_config($vmid, $conf);
7673     });
7674 }
7675
7676 sub convert_iscsi_path {
7677     my ($path) = @_;
7678
7679     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7680         my $portal = $1;
7681         my $target = $2;
7682         my $lun = $3;
7683
7684         my $initiator_name = get_initiator_name();
7685
7686         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7687                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7688     }
7689
7690     die "cannot convert iscsi path '$path', unkown format\n";
7691 }
7692
7693 sub qemu_img_convert {
7694     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7695
7696     my $storecfg = PVE::Storage::config();
7697     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7698     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7699
7700     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7701
7702     my $cachemode;
7703     my $src_path;
7704     my $src_is_iscsi = 0;
7705     my $src_format;
7706
7707     if ($src_storeid) {
7708         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7709         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7710         $src_format = qemu_img_format($src_scfg, $src_volname);
7711         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7712         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7713         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7714     } elsif (-f $src_volid || -b $src_volid) {
7715         $src_path = $src_volid;
7716         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7717             $src_format = $1;
7718         }
7719     }
7720
7721     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7722
7723     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7724     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7725     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7726     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7727
7728     my $cmd = [];
7729     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7730     push @$cmd, '-l', "snapshot.name=$snapname"
7731         if $snapname && $src_format && $src_format eq "qcow2";
7732     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7733     push @$cmd, '-T', $cachemode if defined($cachemode);
7734     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7735
7736     if ($src_is_iscsi) {
7737         push @$cmd, '--image-opts';
7738         $src_path = convert_iscsi_path($src_path);
7739     } elsif ($src_format) {
7740         push @$cmd, '-f', $src_format;
7741     }
7742
7743     if ($dst_is_iscsi) {
7744         push @$cmd, '--target-image-opts';
7745         $dst_path = convert_iscsi_path($dst_path);
7746     } else {
7747         push @$cmd, '-O', $dst_format;
7748     }
7749
7750     push @$cmd, $src_path;
7751
7752     if (!$dst_is_iscsi && $is_zero_initialized) {
7753         push @$cmd, "zeroinit:$dst_path";
7754     } else {
7755         push @$cmd, $dst_path;
7756     }
7757
7758     my $parser = sub {
7759         my $line = shift;
7760         if($line =~ m/\((\S+)\/100\%\)/){
7761             my $percent = $1;
7762             my $transferred = int($size * $percent / 100);
7763             my $total_h = render_bytes($size, 1);
7764             my $transferred_h = render_bytes($transferred, 1);
7765
7766             print "transferred $transferred_h of $total_h ($percent%)\n";
7767         }
7768
7769     };
7770
7771     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7772     my $err = $@;
7773     die "copy failed: $err" if $err;
7774 }
7775
7776 sub qemu_img_format {
7777     my ($scfg, $volname) = @_;
7778
7779     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7780         return $1;
7781     } else {
7782         return "raw";
7783     }
7784 }
7785
7786 sub qemu_drive_mirror {
7787     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7788
7789     $jobs = {} if !$jobs;
7790
7791     my $qemu_target;
7792     my $format;
7793     $jobs->{"drive-$drive"} = {};
7794
7795     if ($dst_volid =~ /^nbd:/) {
7796         $qemu_target = $dst_volid;
7797         $format = "nbd";
7798     } else {
7799         my $storecfg = PVE::Storage::config();
7800         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7801
7802         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7803
7804         $format = qemu_img_format($dst_scfg, $dst_volname);
7805
7806         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7807
7808         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7809     }
7810
7811     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7812     $opts->{format} = $format if $format;
7813
7814     if (defined($src_bitmap)) {
7815         $opts->{sync} = 'incremental';
7816         $opts->{bitmap} = $src_bitmap;
7817         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7818     }
7819
7820     if (defined($bwlimit)) {
7821         $opts->{speed} = $bwlimit * 1024;
7822         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7823     } else {
7824         print "drive mirror is starting for drive-$drive\n";
7825     }
7826
7827     # if a job already runs for this device we get an error, catch it for cleanup
7828     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7829     if (my $err = $@) {
7830         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7831         warn "$@\n" if $@;
7832         die "mirroring error: $err\n";
7833     }
7834
7835     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7836 }
7837
7838 # $completion can be either
7839 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7840 # 'cancel': wait until all jobs are ready, block-job-cancel them
7841 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7842 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7843 sub qemu_drive_mirror_monitor {
7844     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7845
7846     $completion //= 'complete';
7847     $op //= "mirror";
7848
7849     eval {
7850         my $err_complete = 0;
7851
7852         my $starttime = time ();
7853         while (1) {
7854             die "block job ('$op') timed out\n" if $err_complete > 300;
7855
7856             my $stats = mon_cmd($vmid, "query-block-jobs");
7857             my $ctime = time();
7858
7859             my $running_jobs = {};
7860             for my $stat (@$stats) {
7861                 next if $stat->{type} ne $op;
7862                 $running_jobs->{$stat->{device}} = $stat;
7863             }
7864
7865             my $readycounter = 0;
7866
7867             for my $job_id (sort keys %$jobs) {
7868                 my $job = $running_jobs->{$job_id};
7869
7870                 my $vanished = !defined($job);
7871                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7872                 if($complete || ($vanished && $completion eq 'auto')) {
7873                     print "$job_id: $op-job finished\n";
7874                     delete $jobs->{$job_id};
7875                     next;
7876                 }
7877
7878                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7879
7880                 my $busy = $job->{busy};
7881                 my $ready = $job->{ready};
7882                 if (my $total = $job->{len}) {
7883                     my $transferred = $job->{offset} || 0;
7884                     my $remaining = $total - $transferred;
7885                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7886
7887                     my $duration = $ctime - $starttime;
7888                     my $total_h = render_bytes($total, 1);
7889                     my $transferred_h = render_bytes($transferred, 1);
7890
7891                     my $status = sprintf(
7892                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7893                         render_duration($duration),
7894                     );
7895
7896                     if ($ready) {
7897                         if ($busy) {
7898                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7899                         } else {
7900                             $status .= ", ready";
7901                         }
7902                     }
7903                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7904                     $jobs->{$job_id}->{ready} = $ready;
7905                 }
7906
7907                 $readycounter++ if $job->{ready};
7908             }
7909
7910             last if scalar(keys %$jobs) == 0;
7911
7912             if ($readycounter == scalar(keys %$jobs)) {
7913                 print "all '$op' jobs are ready\n";
7914
7915                 # do the complete later (or has already been done)
7916                 last if $completion eq 'skip' || $completion eq 'auto';
7917
7918                 if ($vmiddst && $vmiddst != $vmid) {
7919                     my $agent_running = $qga && qga_check_running($vmid);
7920                     if ($agent_running) {
7921                         print "freeze filesystem\n";
7922                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7923                         warn $@ if $@;
7924                     } else {
7925                         print "suspend vm\n";
7926                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7927                         warn $@ if $@;
7928                     }
7929
7930                     # if we clone a disk for a new target vm, we don't switch the disk
7931                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7932
7933                     if ($agent_running) {
7934                         print "unfreeze filesystem\n";
7935                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7936                         warn $@ if $@;
7937                     } else {
7938                         print "resume vm\n";
7939                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7940                         warn $@ if $@;
7941                     }
7942
7943                     last;
7944                 } else {
7945
7946                     for my $job_id (sort keys %$jobs) {
7947                         # try to switch the disk if source and destination are on the same guest
7948                         print "$job_id: Completing block job_id...\n";
7949
7950                         my $op;
7951                         if ($completion eq 'complete') {
7952                             $op = 'block-job-complete';
7953                         } elsif ($completion eq 'cancel') {
7954                             $op = 'block-job-cancel';
7955                         } else {
7956                             die "invalid completion value: $completion\n";
7957                         }
7958                         eval { mon_cmd($vmid, $op, device => $job_id) };
7959                         if ($@ =~ m/cannot be completed/) {
7960                             print "$job_id: block job cannot be completed, trying again.\n";
7961                             $err_complete++;
7962                         }else {
7963                             print "$job_id: Completed successfully.\n";
7964                             $jobs->{$job_id}->{complete} = 1;
7965                         }
7966                     }
7967                 }
7968             }
7969             sleep 1;
7970         }
7971     };
7972     my $err = $@;
7973
7974     if ($err) {
7975         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7976         die "block job ($op) error: $err";
7977     }
7978 }
7979
7980 sub qemu_blockjobs_cancel {
7981     my ($vmid, $jobs) = @_;
7982
7983     foreach my $job (keys %$jobs) {
7984         print "$job: Cancelling block job\n";
7985         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7986         $jobs->{$job}->{cancel} = 1;
7987     }
7988
7989     while (1) {
7990         my $stats = mon_cmd($vmid, "query-block-jobs");
7991
7992         my $running_jobs = {};
7993         foreach my $stat (@$stats) {
7994             $running_jobs->{$stat->{device}} = $stat;
7995         }
7996
7997         foreach my $job (keys %$jobs) {
7998
7999             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
8000                 print "$job: Done.\n";
8001                 delete $jobs->{$job};
8002             }
8003         }
8004
8005         last if scalar(keys %$jobs) == 0;
8006
8007         sleep 1;
8008     }
8009 }
8010
8011 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8012 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8013 my sub clone_disk_check_io_uring {
8014     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8015
8016     return if !$use_drive_mirror;
8017
8018     # Don't complain when not changing storage.
8019     # Assume if it works for the source, it'll work for the target too.
8020     return if $src_storeid eq $dst_storeid;
8021
8022     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8023     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8024
8025     my $cache_direct = drive_uses_cache_direct($src_drive);
8026
8027     my $src_uses_io_uring;
8028     if ($src_drive->{aio}) {
8029         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8030     } else {
8031         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8032     }
8033
8034     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8035         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8036 }
8037
8038 sub clone_disk {
8039     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8040
8041     my ($vmid, $running) = $source->@{qw(vmid running)};
8042     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8043
8044     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8045     my ($storage, $format) = $dest->@{qw(storage format)};
8046
8047     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8048
8049     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8050         die "cloning from/to EFI disk requires EFI disk\n"
8051             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8052         die "cloning from/to TPM state requires TPM state\n"
8053             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8054
8055         # This would lead to two device nodes in QEMU pointing to the same backing image!
8056         die "cannot change drive name when cloning disk from/to the same VM\n"
8057             if $use_drive_mirror && $vmid == $newvmid;
8058     }
8059
8060     die "cannot move TPM state while VM is running\n"
8061         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8062
8063     my $newvolid;
8064
8065     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8066     print "$src_drivename " if $src_drivename;
8067     print "($drive->{file})\n";
8068
8069     if (!$full) {
8070         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8071         push @$newvollist, $newvolid;
8072     } else {
8073         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8074         my $storeid = $storage || $src_storeid;
8075
8076         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8077
8078         my $name = undef;
8079         my $size = undef;
8080         if (drive_is_cloudinit($drive)) {
8081             $name = "vm-$newvmid-cloudinit";
8082             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8083             if ($scfg->{path}) {
8084                 $name .= ".$dst_format";
8085             }
8086             $snapname = undef;
8087             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8088         } elsif ($dst_drivename eq 'efidisk0') {
8089             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8090         } elsif ($dst_drivename eq 'tpmstate0') {
8091             $dst_format = 'raw';
8092             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8093         } else {
8094             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8095
8096             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8097         }
8098         $newvolid = PVE::Storage::vdisk_alloc(
8099             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8100         );
8101         push @$newvollist, $newvolid;
8102
8103         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8104
8105         if (drive_is_cloudinit($drive)) {
8106             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8107             # if this is the case, we have to complete any block-jobs still there from
8108             # previous drive-mirrors
8109             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8110                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8111             }
8112             goto no_data_clone;
8113         }
8114
8115         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8116         if ($use_drive_mirror) {
8117             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8118                 $completion, $qga, $bwlimit);
8119         } else {
8120             if ($dst_drivename eq 'efidisk0') {
8121                 # the relevant data on the efidisk may be smaller than the source
8122                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8123                 # that is given by the OVMF_VARS.fd
8124                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8125                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8126
8127                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8128
8129                 # better for Ceph if block size is not too small, see bug #3324
8130                 my $bs = 1024*1024;
8131
8132                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8133
8134                 if ($src_format eq 'qcow2' && $snapname) {
8135                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8136                         if !min_version(kvm_user_version(), 6, 2);
8137                     push $cmd->@*, '-l', $snapname;
8138                 }
8139                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8140                 run_command($cmd);
8141             } else {
8142                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8143             }
8144         }
8145     }
8146
8147 no_data_clone:
8148     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8149
8150     my $disk = dclone($drive);
8151     delete $disk->{format};
8152     $disk->{file} = $newvolid;
8153     $disk->{size} = $size if defined($size);
8154
8155     return $disk;
8156 }
8157
8158 sub get_running_qemu_version {
8159     my ($vmid) = @_;
8160     my $res = mon_cmd($vmid, "query-version");
8161     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8162 }
8163
8164 sub qemu_use_old_bios_files {
8165     my ($machine_type) = @_;
8166
8167     return if !$machine_type;
8168
8169     my $use_old_bios_files = undef;
8170
8171     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8172         $machine_type = $1;
8173         $use_old_bios_files = 1;
8174     } else {
8175         my $version = extract_version($machine_type, kvm_user_version());
8176         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8177         # load new efi bios files on migration. So this hack is required to allow
8178         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8179         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8180         $use_old_bios_files = !min_version($version, 2, 4);
8181     }
8182
8183     return ($use_old_bios_files, $machine_type);
8184 }
8185
8186 sub get_efivars_size {
8187     my ($conf, $efidisk) = @_;
8188
8189     my $arch = get_vm_arch($conf);
8190     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8191     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8192     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8193     return -s $ovmf_vars;
8194 }
8195
8196 sub update_efidisk_size {
8197     my ($conf) = @_;
8198
8199     return if !defined($conf->{efidisk0});
8200
8201     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8202     $disk->{size} = get_efivars_size($conf);
8203     $conf->{efidisk0} = print_drive($disk);
8204
8205     return;
8206 }
8207
8208 sub update_tpmstate_size {
8209     my ($conf) = @_;
8210
8211     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8212     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8213     $conf->{tpmstate0} = print_drive($disk);
8214 }
8215
8216 sub create_efidisk($$$$$$$) {
8217     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8218
8219     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8220
8221     my $vars_size_b = -s $ovmf_vars;
8222     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8223     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8224     PVE::Storage::activate_volumes($storecfg, [$volid]);
8225
8226     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8227     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8228
8229     return ($volid, $size/1024);
8230 }
8231
8232 sub vm_iothreads_list {
8233     my ($vmid) = @_;
8234
8235     my $res = mon_cmd($vmid, 'query-iothreads');
8236
8237     my $iothreads = {};
8238     foreach my $iothread (@$res) {
8239         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8240     }
8241
8242     return $iothreads;
8243 }
8244
8245 sub scsihw_infos {
8246     my ($conf, $drive) = @_;
8247
8248     my $maxdev = 0;
8249
8250     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8251         $maxdev = 7;
8252     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8253         $maxdev = 1;
8254     } else {
8255         $maxdev = 256;
8256     }
8257
8258     my $controller = int($drive->{index} / $maxdev);
8259     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8260         ? "virtioscsi"
8261         : "scsihw";
8262
8263     return ($maxdev, $controller, $controller_prefix);
8264 }
8265
8266 sub resolve_dst_disk_format {
8267         my ($storecfg, $storeid, $src_volname, $format) = @_;
8268         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8269
8270         if (!$format) {
8271             # if no target format is specified, use the source disk format as hint
8272             if ($src_volname) {
8273                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8274                 $format = qemu_img_format($scfg, $src_volname);
8275             } else {
8276                 return $defFormat;
8277             }
8278         }
8279
8280         # test if requested format is supported - else use default
8281         my $supported = grep { $_ eq $format } @$validFormats;
8282         $format = $defFormat if !$supported;
8283         return $format;
8284 }
8285
8286 # NOTE: if this logic changes, please update docs & possibly gui logic
8287 sub find_vmstate_storage {
8288     my ($conf, $storecfg) = @_;
8289
8290     # first, return storage from conf if set
8291     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8292
8293     my ($target, $shared, $local);
8294
8295     foreach_storage_used_by_vm($conf, sub {
8296         my ($sid) = @_;
8297         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8298         my $dst = $scfg->{shared} ? \$shared : \$local;
8299         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8300     });
8301
8302     # second, use shared storage where VM has at least one disk
8303     # third, use local storage where VM has at least one disk
8304     # fall back to local storage
8305     $target = $shared // $local // 'local';
8306
8307     return $target;
8308 }
8309
8310 sub generate_uuid {
8311     my ($uuid, $uuid_str);
8312     UUID::generate($uuid);
8313     UUID::unparse($uuid, $uuid_str);
8314     return $uuid_str;
8315 }
8316
8317 sub generate_smbios1_uuid {
8318     return "uuid=".generate_uuid();
8319 }
8320
8321 sub nbd_stop {
8322     my ($vmid) = @_;
8323
8324     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8325 }
8326
8327 sub create_reboot_request {
8328     my ($vmid) = @_;
8329     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8330         or die "failed to create reboot trigger file: $!\n";
8331     close($fh);
8332 }
8333
8334 sub clear_reboot_request {
8335     my ($vmid) = @_;
8336     my $path = "/run/qemu-server/$vmid.reboot";
8337     my $res = 0;
8338
8339     $res = unlink($path);
8340     die "could not remove reboot request for $vmid: $!"
8341         if !$res && $! != POSIX::ENOENT;
8342
8343     return $res;
8344 }
8345
8346 sub bootorder_from_legacy {
8347     my ($conf, $bootcfg) = @_;
8348
8349     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8350     my $bootindex_hash = {};
8351     my $i = 1;
8352     foreach my $o (split(//, $boot)) {
8353         $bootindex_hash->{$o} = $i*100;
8354         $i++;
8355     }
8356
8357     my $bootorder = {};
8358
8359     PVE::QemuConfig->foreach_volume($conf, sub {
8360         my ($ds, $drive) = @_;
8361
8362         if (drive_is_cdrom ($drive, 1)) {
8363             if ($bootindex_hash->{d}) {
8364                 $bootorder->{$ds} = $bootindex_hash->{d};
8365                 $bootindex_hash->{d} += 1;
8366             }
8367         } elsif ($bootindex_hash->{c}) {
8368             $bootorder->{$ds} = $bootindex_hash->{c}
8369                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8370             $bootindex_hash->{c} += 1;
8371         }
8372     });
8373
8374     if ($bootindex_hash->{n}) {
8375         for (my $i = 0; $i < $MAX_NETS; $i++) {
8376             my $netname = "net$i";
8377             next if !$conf->{$netname};
8378             $bootorder->{$netname} = $bootindex_hash->{n};
8379             $bootindex_hash->{n} += 1;
8380         }
8381     }
8382
8383     return $bootorder;
8384 }
8385
8386 # Generate default device list for 'boot: order=' property. Matches legacy
8387 # default boot order, but with explicit device names. This is important, since
8388 # the fallback for when neither 'order' nor the old format is specified relies
8389 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8390 sub get_default_bootdevices {
8391     my ($conf) = @_;
8392
8393     my @ret = ();
8394
8395     # harddisk
8396     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8397     push @ret, $first if $first;
8398
8399     # cdrom
8400     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8401     push @ret, $first if $first;
8402
8403     # network
8404     for (my $i = 0; $i < $MAX_NETS; $i++) {
8405         my $netname = "net$i";
8406         next if !$conf->{$netname};
8407         push @ret, $netname;
8408         last;
8409     }
8410
8411     return \@ret;
8412 }
8413
8414 sub device_bootorder {
8415     my ($conf) = @_;
8416
8417     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8418
8419     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8420
8421     my $bootorder = {};
8422     if (!defined($boot) || $boot->{legacy}) {
8423         $bootorder = bootorder_from_legacy($conf, $boot);
8424     } elsif ($boot->{order}) {
8425         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8426         for my $dev (PVE::Tools::split_list($boot->{order})) {
8427             $bootorder->{$dev} = $i++;
8428         }
8429     }
8430
8431     return $bootorder;
8432 }
8433
8434 sub register_qmeventd_handle {
8435     my ($vmid) = @_;
8436
8437     my $fh;
8438     my $peer = "/var/run/qmeventd.sock";
8439     my $count = 0;
8440
8441     for (;;) {
8442         $count++;
8443         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8444         last if $fh;
8445         if ($! != EINTR && $! != EAGAIN) {
8446             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8447         }
8448         if ($count > 4) {
8449             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8450               . "after $count retries\n";
8451         }
8452         usleep(25000);
8453     }
8454
8455     # send handshake to mark VM as backing up
8456     print $fh to_json({vzdump => {vmid => "$vmid"}});
8457
8458     # return handle to be closed later when inhibit is no longer required
8459     return $fh;
8460 }
8461
8462 # bash completion helper
8463
8464 sub complete_backup_archives {
8465     my ($cmdname, $pname, $cvalue) = @_;
8466
8467     my $cfg = PVE::Storage::config();
8468
8469     my $storeid;
8470
8471     if ($cvalue =~ m/^([^:]+):/) {
8472         $storeid = $1;
8473     }
8474
8475     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8476
8477     my $res = [];
8478     foreach my $id (keys %$data) {
8479         foreach my $item (@{$data->{$id}}) {
8480             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8481             push @$res, $item->{volid} if defined($item->{volid});
8482         }
8483     }
8484
8485     return $res;
8486 }
8487
8488 my $complete_vmid_full = sub {
8489     my ($running) = @_;
8490
8491     my $idlist = vmstatus();
8492
8493     my $res = [];
8494
8495     foreach my $id (keys %$idlist) {
8496         my $d = $idlist->{$id};
8497         if (defined($running)) {
8498             next if $d->{template};
8499             next if $running && $d->{status} ne 'running';
8500             next if !$running && $d->{status} eq 'running';
8501         }
8502         push @$res, $id;
8503
8504     }
8505     return $res;
8506 };
8507
8508 sub complete_vmid {
8509     return &$complete_vmid_full();
8510 }
8511
8512 sub complete_vmid_stopped {
8513     return &$complete_vmid_full(0);
8514 }
8515
8516 sub complete_vmid_running {
8517     return &$complete_vmid_full(1);
8518 }
8519
8520 sub complete_storage {
8521
8522     my $cfg = PVE::Storage::config();
8523     my $ids = $cfg->{ids};
8524
8525     my $res = [];
8526     foreach my $sid (keys %$ids) {
8527         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8528         next if !$ids->{$sid}->{content}->{images};
8529         push @$res, $sid;
8530     }
8531
8532     return $res;
8533 }
8534
8535 sub complete_migration_storage {
8536     my ($cmd, $param, $current_value, $all_args) = @_;
8537
8538     my $targetnode = @$all_args[1];
8539
8540     my $cfg = PVE::Storage::config();
8541     my $ids = $cfg->{ids};
8542
8543     my $res = [];
8544     foreach my $sid (keys %$ids) {
8545         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8546         next if !$ids->{$sid}->{content}->{images};
8547         push @$res, $sid;
8548     }
8549
8550     return $res;
8551 }
8552
8553 sub vm_is_paused {
8554     my ($vmid, $include_suspended) = @_;
8555     my $qmpstatus = eval {
8556         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8557         mon_cmd($vmid, "query-status");
8558     };
8559     warn "$@\n" if $@;
8560     return $qmpstatus && (
8561         $qmpstatus->{status} eq "paused" ||
8562         $qmpstatus->{status} eq "prelaunch" ||
8563         ($include_suspended && $qmpstatus->{status} eq "suspended")
8564     );
8565 }
8566
8567 sub check_volume_storage_type {
8568     my ($storecfg, $vol) = @_;
8569
8570     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8571     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8572     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8573
8574     die "storage '$storeid' does not support content-type '$vtype'\n"
8575         if !$scfg->{content}->{$vtype};
8576
8577     return 1;
8578 }
8579
8580 sub add_nets_bridge_fdb {
8581     my ($conf, $vmid) = @_;
8582
8583     for my $opt (keys %$conf) {
8584         next if $opt !~ m/^net(\d+)$/;
8585         my $iface = "tap${vmid}i$1";
8586         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8587         my $net = parse_net($conf->{$opt}, 1) or next;
8588
8589         my $mac = $net->{macaddr};
8590         if (!$mac) {
8591             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8592                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8593             next;
8594         }
8595
8596         my $bridge = $net->{bridge};
8597         if (!$bridge) {
8598             log_warn("Interface '$iface' not attached to any bridge.");
8599             next;
8600         }
8601         if ($have_sdn) {
8602             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8603         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8604             PVE::Network::add_bridge_fdb($iface, $mac);
8605         }
8606     }
8607 }
8608
8609 sub del_nets_bridge_fdb {
8610     my ($conf, $vmid) = @_;
8611
8612     for my $opt (keys %$conf) {
8613         next if $opt !~ m/^net(\d+)$/;
8614         my $iface = "tap${vmid}i$1";
8615
8616         my $net = parse_net($conf->{$opt}) or next;
8617         my $mac = $net->{macaddr} or next;
8618
8619         my $bridge = $net->{bridge};
8620         if ($have_sdn) {
8621             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8622         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8623             PVE::Network::del_bridge_fdb($iface, $mac);
8624         }
8625     }
8626 }
8627
8628 sub create_ifaces_ipams_ips {
8629     my ($conf, $vmid) = @_;
8630
8631     return if !$have_sdn;
8632
8633     foreach my $opt (keys %$conf) {
8634         if ($opt =~ m/^net(\d+)$/) {
8635             my $value = $conf->{$opt};
8636             my $net = PVE::QemuServer::parse_net($value);
8637             eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8638             warn $@ if $@;
8639         }
8640     }
8641 }
8642
8643 sub delete_ifaces_ipams_ips {
8644     my ($conf, $vmid) = @_;
8645
8646     return if !$have_sdn;
8647
8648     foreach my $opt (keys %$conf) {
8649         if ($opt =~ m/^net(\d+)$/) {
8650             my $net = PVE::QemuServer::parse_net($conf->{$opt});
8651             eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8652             warn $@ if $@;
8653         }
8654     }
8655 }
8656
8657 1;