PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory qw(get_current_memory);
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  62 use PVE::QemuServer::USB;
  63
  64 my $have_sdn;
  65 eval {
  66     require PVE::Network::SDN::Zones;
  67     require PVE::Network::SDN::Vnets;
  68     $have_sdn = 1;
  69 };
  70
  71 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  72 my $OVMF = {
  73     x86_64 => {
  74         '4m-no-smm' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  77         ],
  78         '4m-no-smm-ms' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  81         ],
  82         '4m' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  85         ],
  86         '4m-ms' => [
  87             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  89         ],
  90         # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
  91         # anymore. how can we deperacate this sanely without breaking existing instances, or using
  92         # older backups and snapshot?
  93         default => [
  94             "$EDK2_FW_BASE/OVMF_CODE.fd",
  95             "$EDK2_FW_BASE/OVMF_VARS.fd",
  96         ],
  97     },
  98     aarch64 => {
  99         default => [
 100             "$EDK2_FW_BASE/AAVMF_CODE.fd",
 101             "$EDK2_FW_BASE/AAVMF_VARS.fd",
 102         ],
 103     },
 104 };
 105
 106 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 107
 108 # Note about locking: we use flock on the config file protect against concurent actions.
 109 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 110 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 111 # But you can ignore this kind of lock with the --skiplock flag.
 112
 113 cfs_register_file(
 114     '/qemu-server/',
 115     \&parse_vm_config,
 116     \&write_vm_config
 117 );
 118
 119 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 120     description => "Some command save/restore state from this location.",
 121     type => 'string',
 122     maxLength => 128,
 123     optional => 1,
 124 });
 125
 126 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 127         description => "Specifies the QEMU machine type.",
 128         type => 'string',
 129         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 130         maxLength => 40,
 131         optional => 1,
 132 });
 133
 134 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 135 my $nodename_cache;
 136 sub nodename {
 137     $nodename_cache //= PVE::INotify::nodename();
 138     return $nodename_cache;
 139 }
 140
 141 my $watchdog_fmt = {
 142     model => {
 143         default_key => 1,
 144         type => 'string',
 145         enum => [qw(i6300esb ib700)],
 146         description => "Watchdog type to emulate.",
 147         default => 'i6300esb',
 148         optional => 1,
 149     },
 150     action => {
 151         type => 'string',
 152         enum => [qw(reset shutdown poweroff pause debug none)],
 153         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 154         optional => 1,
 155     },
 156 };
 157 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 158
 159 my $agent_fmt = {
 160     enabled => {
 161         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 162         type => 'boolean',
 163         default => 0,
 164         default_key => 1,
 165     },
 166     fstrim_cloned_disks => {
 167         description => "Run fstrim after moving a disk or migrating the VM.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0,
 171     },
 172     'freeze-fs-on-backup' => {
 173         description => "Freeze/thaw guest filesystems on backup for consistency.",
 174         type => 'boolean',
 175         optional => 1,
 176         default => 1,
 177     },
 178     type => {
 179         description => "Select the agent type",
 180         type => 'string',
 181         default => 'virtio',
 182         optional => 1,
 183         enum => [qw(virtio isa)],
 184     },
 185 };
 186
 187 my $vga_fmt = {
 188     type => {
 189         description => "Select the VGA type.",
 190         type => 'string',
 191         default => 'std',
 192         optional => 1,
 193         default_key => 1,
 194         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 195     },
 196     memory => {
 197         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 198         type => 'integer',
 199         optional => 1,
 200         minimum => 4,
 201         maximum => 512,
 202     },
 203     clipboard => {
 204         description => 'Enable a specific clipboard. If not set, depending on the display type the'
 205             .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
 206         type => 'string',
 207         enum => ['vnc'],
 208         optional => 1,
 209     },
 210 };
 211
 212 my $ivshmem_fmt = {
 213     size => {
 214         type => 'integer',
 215         minimum => 1,
 216         description => "The size of the file in MB.",
 217     },
 218     name => {
 219         type => 'string',
 220         pattern => '[a-zA-Z0-9\-]+',
 221         optional => 1,
 222         format_description => 'string',
 223         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 224     },
 225 };
 226
 227 my $audio_fmt = {
 228     device => {
 229         type => 'string',
 230         enum => [qw(ich9-intel-hda intel-hda AC97)],
 231         description =>  "Configure an audio device."
 232     },
 233     driver =>  {
 234         type => 'string',
 235         enum => ['spice', 'none'],
 236         default => 'spice',
 237         optional => 1,
 238         description => "Driver backend for the audio device."
 239     },
 240 };
 241
 242 my $spice_enhancements_fmt = {
 243     foldersharing => {
 244         type => 'boolean',
 245         optional => 1,
 246         default => '0',
 247         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 248     },
 249     videostreaming =>  {
 250         type => 'string',
 251         enum => ['off', 'all', 'filter'],
 252         default => 'off',
 253         optional => 1,
 254         description => "Enable video streaming. Uses compression for detected video streams."
 255     },
 256 };
 257
 258 my $rng_fmt = {
 259     source => {
 260         type => 'string',
 261         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 262         default_key => 1,
 263         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 264             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 265             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 266             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 267             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 268             ." a hardware RNG from the host.",
 269     },
 270     max_bytes => {
 271         type => 'integer',
 272         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 273             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 274             ." `0` to disable limiting (potentially dangerous!).",
 275         optional => 1,
 276
 277         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 278         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 279         # reading from /dev/urandom
 280         default => 1024,
 281     },
 282     period => {
 283         type => 'integer',
 284         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 285             ." the guest to retrieve another 'max_bytes' of entropy.",
 286         optional => 1,
 287         default => 1000,
 288     },
 289 };
 290
 291 my $meta_info_fmt = {
 292     'ctime' => {
 293         type => 'integer',
 294         description => "The guest creation timestamp as UNIX epoch time",
 295         minimum => 0,
 296         optional => 1,
 297     },
 298     'creation-qemu' => {
 299         type => 'string',
 300         description => "The QEMU (machine) version from the time this VM was created.",
 301         pattern => '\d+(\.\d+)+',
 302         optional => 1,
 303     },
 304 };
 305
 306 my $confdesc = {
 307     onboot => {
 308         optional => 1,
 309         type => 'boolean',
 310         description => "Specifies whether a VM will be started during system bootup.",
 311         default => 0,
 312     },
 313     autostart => {
 314         optional => 1,
 315         type => 'boolean',
 316         description => "Automatic restart after crash (currently ignored).",
 317         default => 0,
 318     },
 319     hotplug => {
 320         optional => 1,
 321         type => 'string', format => 'pve-hotplug-features',
 322         description => "Selectively enable hotplug features. This is a comma separated list of"
 323             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 324             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 325             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 326             ." windows > 7.",
 327         default => 'network,disk,usb',
 328     },
 329     reboot => {
 330         optional => 1,
 331         type => 'boolean',
 332         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 333         default => 1,
 334     },
 335     lock => {
 336         optional => 1,
 337         type => 'string',
 338         description => "Lock/unlock the VM.",
 339         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 340     },
 341     cpulimit => {
 342         optional => 1,
 343         type => 'number',
 344         description => "Limit of CPU usage.",
 345         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 346             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 347         minimum => 0,
 348         maximum => 128,
 349         default => 0,
 350     },
 351     cpuunits => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 355         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 356             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 357             ." weights of all the other running VMs.",
 358         minimum => 1,
 359         maximum => 262144,
 360         default => 'cgroup v1: 1024, cgroup v2: 100',
 361     },
 362     memory => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Memory properties.",
 366         format => $PVE::QemuServer::Memory::memory_fmt
 367     },
 368     balloon => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 372         minimum => 0,
 373     },
 374     shares => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 378             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 379             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 380         minimum => 0,
 381         maximum => 50000,
 382         default => 1000,
 383     },
 384     keyboard => {
 385         optional => 1,
 386         type => 'string',
 387         description => "Keyboard layout for VNC server. This option is generally not required and"
 388         ." is often better handled from within the guest OS.",
 389         enum => PVE::Tools::kvmkeymaplist(),
 390         default => undef,
 391     },
 392     name => {
 393         optional => 1,
 394         type => 'string', format => 'dns-name',
 395         description => "Set a name for the VM. Only used on the configuration web interface.",
 396     },
 397     scsihw => {
 398         optional => 1,
 399         type => 'string',
 400         description => "SCSI controller model",
 401         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 402         default => 'lsi',
 403     },
 404     description => {
 405         optional => 1,
 406         type => 'string',
 407         description => "Description for the VM. Shown in the web-interface VM's summary."
 408             ." This is saved as comment inside the configuration file.",
 409         maxLength => 1024 * 8,
 410     },
 411     ostype => {
 412         optional => 1,
 413         type => 'string',
 414         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 415         description => "Specify guest operating system.",
 416         verbose_description => <<EODESC,
 417 Specify guest operating system. This is used to enable special
 418 optimization/features for specific operating systems:
 419
 420 [horizontal]
 421 other;; unspecified OS
 422 wxp;; Microsoft Windows XP
 423 w2k;; Microsoft Windows 2000
 424 w2k3;; Microsoft Windows 2003
 425 w2k8;; Microsoft Windows 2008
 426 wvista;; Microsoft Windows Vista
 427 win7;; Microsoft Windows 7
 428 win8;; Microsoft Windows 8/2012/2012r2
 429 win10;; Microsoft Windows 10/2016/2019
 430 win11;; Microsoft Windows 11/2022
 431 l24;; Linux 2.4 Kernel
 432 l26;; Linux 2.6 - 6.X Kernel
 433 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 434 EODESC
 435     },
 436     boot => {
 437         optional => 1,
 438         type => 'string', format => 'pve-qm-boot',
 439         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 440             ." key or 'legacy=' is deprecated.",
 441     },
 442     bootdisk => {
 443         optional => 1,
 444         type => 'string', format => 'pve-qm-bootdisk',
 445         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 446         pattern => '(ide|sata|scsi|virtio)\d+',
 447     },
 448     smp => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "The number of CPUs. Please use option -sockets instead.",
 452         minimum => 1,
 453         default => 1,
 454     },
 455     sockets => {
 456         optional => 1,
 457         type => 'integer',
 458         description => "The number of CPU sockets.",
 459         minimum => 1,
 460         default => 1,
 461     },
 462     cores => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "The number of cores per socket.",
 466         minimum => 1,
 467         default => 1,
 468     },
 469     numa => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable NUMA.",
 473         default => 0,
 474     },
 475     hugepages => {
 476         optional => 1,
 477         type => 'string',
 478         description => "Enable/disable hugepages memory.",
 479         enum => [qw(any 2 1024)],
 480     },
 481     keephugepages => {
 482         optional => 1,
 483         type => 'boolean',
 484         default => 0,
 485         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 486             ." after VM shutdown and can be used for subsequent starts.",
 487     },
 488     vcpus => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Number of hotplugged vcpus.",
 492         minimum => 1,
 493         default => 0,
 494     },
 495     acpi => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable ACPI.",
 499         default => 1,
 500     },
 501     agent => {
 502         optional => 1,
 503         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 504         type => 'string',
 505         format => $agent_fmt,
 506     },
 507     kvm => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable KVM hardware virtualization.",
 511         default => 1,
 512     },
 513     tdf => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable time drift fix.",
 517         default => 0,
 518     },
 519     localtime => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 523             ." the `ostype` indicates a Microsoft Windows OS.",
 524     },
 525     freeze => {
 526         optional => 1,
 527         type => 'boolean',
 528         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 529     },
 530     vga => {
 531         optional => 1,
 532         type => 'string', format => $vga_fmt,
 533         description => "Configure the VGA hardware.",
 534         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 535             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 536             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 537             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 538             ." display server. For win* OS you can select how many independent displays you want,"
 539             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 540             ." using a serial device as terminal.",
 541     },
 542     watchdog => {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-watchdog',
 545         description => "Create a virtual hardware watchdog device.",
 546         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 547             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 548             ." else the watchdog will reset the guest (or execute the respective action specified)",
 549     },
 550     startdate => {
 551         optional => 1,
 552         type => 'string',
 553         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 554         description => "Set the initial date of the real time clock. Valid format for date are:"
 555             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 556         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 557         default => 'now',
 558     },
 559     startup =>  get_standard_option('pve-startup-order'),
 560     template => {
 561         optional => 1,
 562         type => 'boolean',
 563         description => "Enable/disable Template.",
 564         default => 0,
 565     },
 566     args => {
 567         optional => 1,
 568         type => 'string',
 569         description => "Arbitrary arguments passed to kvm.",
 570         verbose_description => <<EODESCR,
 571 Arbitrary arguments passed to kvm, for example:
 572
 573 args: -no-reboot -smbios 'type=0,vendor=FOO'
 574
 575 NOTE: this option is for experts only.
 576 EODESCR
 577     },
 578     tablet => {
 579         optional => 1,
 580         type => 'boolean',
 581         default => 1,
 582         description => "Enable/disable the USB tablet device.",
 583         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 584             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 585             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 586             ." may consider disabling this to save some context switches. This is turned off by"
 587             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 588     },
 589     migrate_speed => {
 590         optional => 1,
 591         type => 'integer',
 592         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 593         minimum => 0,
 594         default => 0,
 595     },
 596     migrate_downtime => {
 597         optional => 1,
 598         type => 'number',
 599         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 600         minimum => 0,
 601         default => 0.1,
 602     },
 603     cdrom => {
 604         optional => 1,
 605         type => 'string', format => 'pve-qm-ide',
 606         typetext => '<volume>',
 607         description => "This is an alias for option -ide2",
 608     },
 609     cpu => {
 610         optional => 1,
 611         description => "Emulated CPU type.",
 612         type => 'string',
 613         format => 'pve-vm-cpu-conf',
 614     },
 615     parent => get_standard_option('pve-snapshot-name', {
 616         optional => 1,
 617         description => "Parent snapshot name. This is used internally, and should not be modified.",
 618     }),
 619     snaptime => {
 620         optional => 1,
 621         description => "Timestamp for snapshots.",
 622         type => 'integer',
 623         minimum => 0,
 624     },
 625     vmstate => {
 626         optional => 1,
 627         type => 'string', format => 'pve-volume-id',
 628         description => "Reference to a volume which stores the VM state. This is used internally"
 629             ." for snapshots.",
 630     },
 631     vmstatestorage => get_standard_option('pve-storage-id', {
 632         description => "Default storage for VM state volumes/files.",
 633         optional => 1,
 634     }),
 635     runningmachine => get_standard_option('pve-qemu-machine', {
 636         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 637             ." for snapshots.",
 638     }),
 639     runningcpu => {
 640         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 641             ." internally for snapshots.",
 642         optional => 1,
 643         type => 'string',
 644         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 645         format_description => 'QEMU -cpu parameter'
 646     },
 647     machine => get_standard_option('pve-qemu-machine'),
 648     arch => {
 649         description => "Virtual processor architecture. Defaults to the host.",
 650         optional => 1,
 651         type => 'string',
 652         enum => [qw(x86_64 aarch64)],
 653     },
 654     smbios1 => {
 655         description => "Specify SMBIOS type 1 fields.",
 656         type => 'string', format => 'pve-qm-smbios1',
 657         maxLength => 512,
 658         optional => 1,
 659     },
 660     protection => {
 661         optional => 1,
 662         type => 'boolean',
 663         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 664             ." remove disk operations.",
 665         default => 0,
 666     },
 667     bios => {
 668         optional => 1,
 669         type => 'string',
 670         enum => [ qw(seabios ovmf) ],
 671         description => "Select BIOS implementation.",
 672         default => 'seabios',
 673     },
 674     vmgenid => {
 675         type => 'string',
 676         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 677         format_description => 'UUID',
 678         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 679             ." to disable explicitly.",
 680         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 681             ." value identifier to the guest OS. This allows to notify the guest operating system"
 682             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 683             ." execution or creation from a template). The guest operating system notices the"
 684             ." change, and is then able to react as appropriate by marking its copies of"
 685             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 686             ."Note that auto-creation only works when done through API/CLI create or update methods"
 687             .", but not when manually editing the config file.",
 688         default => "1 (autogenerated)",
 689         optional => 1,
 690     },
 691     hookscript => {
 692         type => 'string',
 693         format => 'pve-volume-id',
 694         optional => 1,
 695         description => "Script that will be executed during various steps in the vms lifetime.",
 696     },
 697     ivshmem => {
 698         type => 'string',
 699         format => $ivshmem_fmt,
 700         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 701             ." the host.",
 702         optional => 1,
 703     },
 704     audio0 => {
 705         type => 'string',
 706         format => $audio_fmt,
 707         description => "Configure a audio device, useful in combination with QXL/Spice.",
 708         optional => 1
 709     },
 710     spice_enhancements => {
 711         type => 'string',
 712         format => $spice_enhancements_fmt,
 713         description => "Configure additional enhancements for SPICE.",
 714         optional => 1
 715     },
 716     tags => {
 717         type => 'string', format => 'pve-tag-list',
 718         description => 'Tags of the VM. This is only meta information.',
 719         optional => 1,
 720     },
 721     rng0 => {
 722         type => 'string',
 723         format => $rng_fmt,
 724         description => "Configure a VirtIO-based Random Number Generator.",
 725         optional => 1,
 726     },
 727     meta => {
 728         type => 'string',
 729         format => $meta_info_fmt,
 730         description => "Some (read-only) meta-information about this guest.",
 731         optional => 1,
 732     },
 733     affinity => {
 734         type => 'string', format => 'pve-cpuset',
 735         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 736         optional => 1,
 737     },
 738 };
 739
 740 my $cicustom_fmt = {
 741     meta => {
 742         type => 'string',
 743         optional => 1,
 744         description => 'Specify a custom file containing all meta data passed to the VM via"
 745             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 746         format => 'pve-volume-id',
 747         format_description => 'volume',
 748     },
 749     network => {
 750         type => 'string',
 751         optional => 1,
 752         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 753         format => 'pve-volume-id',
 754         format_description => 'volume',
 755     },
 756     user => {
 757         type => 'string',
 758         optional => 1,
 759         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 760         format => 'pve-volume-id',
 761         format_description => 'volume',
 762     },
 763     vendor => {
 764         type => 'string',
 765         optional => 1,
 766         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 767         format => 'pve-volume-id',
 768         format_description => 'volume',
 769     },
 770 };
 771 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 772
 773 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 774 my $confdesc_cloudinit = {
 775     citype => {
 776         optional => 1,
 777         type => 'string',
 778         description => 'Specifies the cloud-init configuration format. The default depends on the'
 779             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 780             .' and `configdrive2` for windows.',
 781         enum => ['configdrive2', 'nocloud', 'opennebula'],
 782     },
 783     ciuser => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: User name to change ssh keys and password for instead of the"
 787             ." image's configured default user.",
 788     },
 789     cipassword => {
 790         optional => 1,
 791         type => 'string',
 792         description => 'cloud-init: Password to assign the user. Using this is generally not'
 793             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 794             .' support hashed passwords.',
 795     },
 796     ciupgrade => {
 797         optional => 1,
 798         type => 'boolean',
 799         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 800         default => 1,
 801     },
 802     cicustom => {
 803         optional => 1,
 804         type => 'string',
 805         description => 'cloud-init: Specify custom files to replace the automatically generated'
 806             .' ones at start.',
 807         format => 'pve-qm-cicustom',
 808     },
 809     searchdomain => {
 810         optional => 1,
 811         type => 'string',
 812         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 813             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 814             .' are set.',
 815     },
 816     nameserver => {
 817         optional => 1,
 818         type => 'string', format => 'address-list',
 819         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 820             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 821             .' are set.',
 822     },
 823     sshkeys => {
 824         optional => 1,
 825         type => 'string',
 826         format => 'urlencoded',
 827         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 828     },
 829 };
 830
 831 # what about other qemu settings ?
 832 #cpu => 'string',
 833 #machine => 'string',
 834 #fda => 'file',
 835 #fdb => 'file',
 836 #mtdblock => 'file',
 837 #sd => 'file',
 838 #pflash => 'file',
 839 #snapshot => 'bool',
 840 #bootp => 'file',
 841 ##tftp => 'dir',
 842 ##smb => 'dir',
 843 #kernel => 'file',
 844 #append => 'string',
 845 #initrd => 'file',
 846 ##soundhw => 'string',
 847
 848 while (my ($k, $v) = each %$confdesc) {
 849     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 850 }
 851
 852 my $MAX_NETS = 32;
 853 my $MAX_SERIAL_PORTS = 4;
 854 my $MAX_PARALLEL_PORTS = 3;
 855
 856 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 857     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 858 }
 859
 860 my $nic_model_list = [
 861     'e1000',
 862     'e1000-82540em',
 863     'e1000-82544gc',
 864     'e1000-82545em',
 865     'e1000e',
 866     'i82551',
 867     'i82557b',
 868     'i82559er',
 869     'ne2k_isa',
 870     'ne2k_pci',
 871     'pcnet',
 872     'rtl8139',
 873     'virtio',
 874     'vmxnet3',
 875 ];
 876 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 877
 878 my $net_fmt_bridge_descr = <<__EOD__;
 879 Bridge to attach the network device to. The Proxmox VE standard bridge
 880 is called 'vmbr0'.
 881
 882 If you do not specify a bridge, we create a kvm user (NATed) network
 883 device, which provides DHCP and DNS services. The following addresses
 884 are used:
 885
 886  10.0.2.2   Gateway
 887  10.0.2.3   DNS Server
 888  10.0.2.4   SMB Server
 889
 890 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 891 __EOD__
 892
 893 my $net_fmt = {
 894     macaddr  => get_standard_option('mac-addr', {
 895         description => "MAC address. That address must be unique withing your network. This is"
 896             ." automatically generated if not specified.",
 897     }),
 898     model => {
 899         type => 'string',
 900         description => "Network Card Model. The 'virtio' model provides the best performance with"
 901             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 902             ." best to use 'e1000'.",
 903         enum => $nic_model_list,
 904         default_key => 1,
 905     },
 906     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 907     bridge => get_standard_option('pve-bridge-id', {
 908         description => $net_fmt_bridge_descr,
 909         optional => 1,
 910     }),
 911     queues => {
 912         type => 'integer',
 913         minimum => 0, maximum => 64,
 914         description => 'Number of packet queues to be used on the device.',
 915         optional => 1,
 916     },
 917     rate => {
 918         type => 'number',
 919         minimum => 0,
 920         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 921         optional => 1,
 922     },
 923     tag => {
 924         type => 'integer',
 925         minimum => 1, maximum => 4094,
 926         description => 'VLAN tag to apply to packets on this interface.',
 927         optional => 1,
 928     },
 929     trunks => {
 930         type => 'string',
 931         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 932         description => 'VLAN trunks to pass through this interface.',
 933         format_description => 'vlanid[;vlanid...]',
 934         optional => 1,
 935     },
 936     firewall => {
 937         type => 'boolean',
 938         description => 'Whether this interface should be protected by the firewall.',
 939         optional => 1,
 940     },
 941     link_down => {
 942         type => 'boolean',
 943         description => 'Whether this interface should be disconnected (like pulling the plug).',
 944         optional => 1,
 945     },
 946     mtu => {
 947         type => 'integer',
 948         minimum => 1, maximum => 65520,
 949         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 950         optional => 1,
 951     },
 952 };
 953
 954 my $netdesc = {
 955     optional => 1,
 956     type => 'string', format => $net_fmt,
 957     description => "Specify network devices.",
 958 };
 959
 960 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 961
 962 my $ipconfig_fmt = {
 963     ip => {
 964         type => 'string',
 965         format => 'pve-ipv4-config',
 966         format_description => 'IPv4Format/CIDR',
 967         description => 'IPv4 address in CIDR format.',
 968         optional => 1,
 969         default => 'dhcp',
 970     },
 971     gw => {
 972         type => 'string',
 973         format => 'ipv4',
 974         format_description => 'GatewayIPv4',
 975         description => 'Default gateway for IPv4 traffic.',
 976         optional => 1,
 977         requires => 'ip',
 978     },
 979     ip6 => {
 980         type => 'string',
 981         format => 'pve-ipv6-config',
 982         format_description => 'IPv6Format/CIDR',
 983         description => 'IPv6 address in CIDR format.',
 984         optional => 1,
 985         default => 'dhcp',
 986     },
 987     gw6 => {
 988         type => 'string',
 989         format => 'ipv6',
 990         format_description => 'GatewayIPv6',
 991         description => 'Default gateway for IPv6 traffic.',
 992         optional => 1,
 993         requires => 'ip6',
 994     },
 995 };
 996 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 997 my $ipconfigdesc = {
 998     optional => 1,
 999     type => 'string', format => 'pve-qm-ipconfig',
1000     description => <<'EODESCR',
1001 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1002
1003 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1004
1005 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1006 gateway should be provided.
1007 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1008 cloud-init 19.4 or newer.
1009
1010 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1011 dhcp on IPv4.
1012 EODESCR
1013 };
1014 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1015
1016 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1017     $confdesc->{"net$i"} = $netdesc;
1018     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1019 }
1020
1021 foreach my $key (keys %$confdesc_cloudinit) {
1022     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1023 }
1024
1025 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1026 sub pve_verify_cpuset {
1027     my ($set_text, $noerr) = @_;
1028
1029     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1030
1031     if ($@) {
1032         return if $noerr;
1033         die "unable to parse cpuset option\n";
1034     }
1035
1036     return PVE::CpuSet->new($members)->short_string();
1037 }
1038
1039 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1040 sub verify_volume_id_or_qm_path {
1041     my ($volid, $noerr) = @_;
1042
1043     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1044
1045     return verify_volume_id_or_absolute_path($volid, $noerr);
1046 }
1047
1048 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1049 sub verify_volume_id_or_absolute_path {
1050     my ($volid, $noerr) = @_;
1051
1052     return $volid if $volid =~ m|^/|;
1053
1054     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1055     if ($@) {
1056         return if $noerr;
1057         die $@;
1058     }
1059     return $volid;
1060 }
1061
1062 my $serialdesc = {
1063         optional => 1,
1064         type => 'string',
1065         pattern => '(/dev/.+|socket)',
1066         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1067         verbose_description =>  <<EODESCR,
1068 Create a serial device inside the VM (n is 0 to 3), and pass through a
1069 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1070 host side (use 'qm terminal' to open a terminal connection).
1071
1072 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1073 use with special care.
1074
1075 CAUTION: Experimental! User reported problems with this option.
1076 EODESCR
1077 };
1078
1079 my $paralleldesc= {
1080         optional => 1,
1081         type => 'string',
1082         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1083         description =>  "Map host parallel devices (n is 0 to 2).",
1084         verbose_description =>  <<EODESCR,
1085 Map host parallel devices (n is 0 to 2).
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1095     $confdesc->{"parallel$i"} = $paralleldesc;
1096 }
1097
1098 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1099     $confdesc->{"serial$i"} = $serialdesc;
1100 }
1101
1102 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1103     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1104 }
1105
1106 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1107     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1108 }
1109
1110 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1111     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1112 }
1113
1114 my $boot_fmt = {
1115     legacy => {
1116         optional => 1,
1117         default_key => 1,
1118         type => 'string',
1119         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1120                      . " Deprecated, use 'order=' instead.",
1121         pattern => '[acdn]{1,4}',
1122         format_description => "[acdn]{1,4}",
1123
1124         # note: this is also the fallback if boot: is not given at all
1125         default => 'cdn',
1126     },
1127     order => {
1128         optional => 1,
1129         type => 'string',
1130         format => 'pve-qm-bootdev-list',
1131         format_description => "device[;device...]",
1132         description => <<EODESC,
1133 The guest will attempt to boot from devices in the order they appear here.
1134
1135 Disks, optical drives and passed-through storage USB devices will be directly
1136 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1137 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1138
1139 Note that only devices in this list will be marked as bootable and thus loaded
1140 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1141 (e.g. software-raid), you need to specify all of them here.
1142
1143 Overrides the deprecated 'legacy=[acdn]*' value when given.
1144 EODESC
1145     },
1146 };
1147 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1148
1149 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1150 sub verify_bootdev {
1151     my ($dev, $noerr) = @_;
1152
1153     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1154     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1155
1156     my $check = sub {
1157         my ($base) = @_;
1158         return 0 if $dev !~ m/^$base\d+$/;
1159         return 0 if !$confdesc->{$dev};
1160         return 1;
1161     };
1162
1163     return $dev if $check->("net");
1164     return $dev if $check->("usb");
1165     return $dev if $check->("hostpci");
1166
1167     return if $noerr;
1168     die "invalid boot device '$dev'\n";
1169 }
1170
1171 sub print_bootorder {
1172     my ($devs) = @_;
1173     return "" if !@$devs;
1174     my $data = { order => join(';', @$devs) };
1175     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1176 }
1177
1178 my $kvm_api_version = 0;
1179
1180 sub kvm_version {
1181     return $kvm_api_version if $kvm_api_version;
1182
1183     open my $fh, '<', '/dev/kvm' or return;
1184
1185     # 0xae00 => KVM_GET_API_VERSION
1186     $kvm_api_version = ioctl($fh, 0xae00, 0);
1187     close($fh);
1188
1189     return $kvm_api_version;
1190 }
1191
1192 my $kvm_user_version = {};
1193 my $kvm_mtime = {};
1194
1195 sub kvm_user_version {
1196     my ($binary) = @_;
1197
1198     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1199     my $st = stat($binary);
1200
1201     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1202     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1203         $cachedmtime == $st->mtime;
1204
1205     $kvm_user_version->{$binary} = 'unknown';
1206     $kvm_mtime->{$binary} = $st->mtime;
1207
1208     my $code = sub {
1209         my $line = shift;
1210         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1211             $kvm_user_version->{$binary} = $2;
1212         }
1213     };
1214
1215     eval { run_command([$binary, '--version'], outfunc => $code); };
1216     warn $@ if $@;
1217
1218     return $kvm_user_version->{$binary};
1219
1220 }
1221 my sub extract_version {
1222     my ($machine_type, $version) = @_;
1223     $version = kvm_user_version() if !defined($version);
1224     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1225 }
1226
1227 sub kernel_has_vhost_net {
1228     return -c '/dev/vhost-net';
1229 }
1230
1231 sub option_exists {
1232     my $key = shift;
1233     return defined($confdesc->{$key});
1234 }
1235
1236 my $cdrom_path;
1237 sub get_cdrom_path {
1238
1239     return $cdrom_path if defined($cdrom_path);
1240
1241     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1242
1243     if (!defined($cdrom_path)) {
1244         log_warn("no physical CD-ROM available, ignoring");
1245         $cdrom_path = '';
1246     }
1247
1248     return $cdrom_path;
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269      if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1314             if !$vtype;
1315         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1316         verify_media_type($opt, $vtype, $drive->{media});
1317         $drive->{file} = $volid;
1318     }
1319
1320     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1321 }
1322
1323 sub parse_hotplug_features {
1324     my ($data) = @_;
1325
1326     my $res = {};
1327
1328     return $res if $data eq '0';
1329
1330     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1331
1332     foreach my $feature (PVE::Tools::split_list($data)) {
1333         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1334             $res->{$1} = 1;
1335         } else {
1336             die "invalid hotplug feature '$feature'\n";
1337         }
1338     }
1339     return $res;
1340 }
1341
1342 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1343 sub pve_verify_hotplug_features {
1344     my ($value, $noerr) = @_;
1345
1346     return $value if parse_hotplug_features($value);
1347
1348     return if $noerr;
1349
1350     die "unable to parse hotplug option\n";
1351 }
1352
1353 sub assert_clipboard_config {
1354     my ($vga) = @_;
1355
1356     my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1357
1358     if (
1359         $vga->{'clipboard'}
1360         && $vga->{'clipboard'} eq 'vnc'
1361         && $vga->{type}
1362         && $vga->{type} !~ $clipboard_regex
1363     ) {
1364         die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1365     }
1366 }
1367
1368 sub scsi_inquiry {
1369     my($fh, $noerr) = @_;
1370
1371     my $SG_IO = 0x2285;
1372     my $SG_GET_VERSION_NUM = 0x2282;
1373
1374     my $versionbuf = "\x00" x 8;
1375     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1376     if (!$ret) {
1377         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1378         return;
1379     }
1380     my $version = unpack("I", $versionbuf);
1381     if ($version < 30000) {
1382         die "scsi generic interface too old\n"  if !$noerr;
1383         return;
1384     }
1385
1386     my $buf = "\x00" x 36;
1387     my $sensebuf = "\x00" x 8;
1388     my $cmd = pack("C x3 C x1", 0x12, 36);
1389
1390     # see /usr/include/scsi/sg.h
1391     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1392
1393     my $packet = pack(
1394         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1395     );
1396
1397     $ret = ioctl($fh, $SG_IO, $packet);
1398     if (!$ret) {
1399         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1400         return;
1401     }
1402
1403     my @res = unpack($sg_io_hdr_t, $packet);
1404     if ($res[17] || $res[18]) {
1405         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1406         return;
1407     }
1408
1409     my $res = {};
1410     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1411
1412     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1413     $res->{type} &= 0x1F;
1414
1415     return $res;
1416 }
1417
1418 sub path_is_scsi {
1419     my ($path) = @_;
1420
1421     my $fh = IO::File->new("+<$path") || return;
1422     my $res = scsi_inquiry($fh, 1);
1423     close($fh);
1424
1425     return $res;
1426 }
1427
1428 sub print_tabletdevice_full {
1429     my ($conf, $arch) = @_;
1430
1431     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1432
1433     # we use uhci for old VMs because tablet driver was buggy in older qemu
1434     my $usbbus;
1435     if ($q35 || $arch eq 'aarch64') {
1436         $usbbus = 'ehci';
1437     } else {
1438         $usbbus = 'uhci';
1439     }
1440
1441     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1442 }
1443
1444 sub print_keyboarddevice_full {
1445     my ($conf, $arch) = @_;
1446
1447     return if $arch ne 'aarch64';
1448
1449     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1450 }
1451
1452 my sub get_drive_id {
1453     my ($drive) = @_;
1454     return "$drive->{interface}$drive->{index}";
1455 }
1456
1457 sub print_drivedevice_full {
1458     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1459
1460     my $device = '';
1461     my $maxdev = 0;
1462
1463     my $drive_id = get_drive_id($drive);
1464     if ($drive->{interface} eq 'virtio') {
1465         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1466         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1467         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1468     } elsif ($drive->{interface} eq 'scsi') {
1469
1470         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1471         my $unit = $drive->{index} % $maxdev;
1472         my $devicetype = 'hd';
1473         my $path = '';
1474         if (drive_is_cdrom($drive)) {
1475             $devicetype = 'cd';
1476         } else {
1477             if ($drive->{file} =~ m|^/|) {
1478                 $path = $drive->{file};
1479                 if (my $info = path_is_scsi($path)) {
1480                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1481                         $devicetype = 'block';
1482                     } elsif ($info->{type} == 1) { # tape
1483                         $devicetype = 'generic';
1484                     }
1485                 }
1486             } else {
1487                  $path = PVE::Storage::path($storecfg, $drive->{file});
1488             }
1489
1490             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1491             my $version = extract_version($machine_type, kvm_user_version());
1492             if ($path =~ m/^iscsi\:\/\// &&
1493                !min_version($version, 4, 1)) {
1494                 $devicetype = 'generic';
1495             }
1496         }
1497
1498         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1499             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1500         } else {
1501             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1502                 .",lun=$drive->{index}";
1503         }
1504         $device .= ",drive=drive-$drive_id,id=$drive_id";
1505
1506         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1507             $device .= ",rotation_rate=1";
1508         }
1509         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1510
1511     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1512         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1513         my $controller = int($drive->{index} / $maxdev);
1514         my $unit = $drive->{index} % $maxdev;
1515
1516         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1517         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1518         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1519         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1520         # were before. Move odd ones up by 2 where they don't clash.
1521         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1522             $controller += 2 * ($unit % 2);
1523             $unit = 0;
1524         }
1525
1526         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1527
1528         $device = "ide-$devicetype";
1529         if ($drive->{interface} eq 'ide') {
1530             $device .= ",bus=ide.$controller,unit=$unit";
1531         } else {
1532             $device .= ",bus=ahci$controller.$unit";
1533         }
1534         $device .= ",drive=drive-$drive_id,id=$drive_id";
1535
1536         if ($devicetype eq 'hd') {
1537             if (my $model = $drive->{model}) {
1538                 $model = URI::Escape::uri_unescape($model);
1539                 $device .= ",model=$model";
1540             }
1541             if ($drive->{ssd}) {
1542                 $device .= ",rotation_rate=1";
1543             }
1544         }
1545         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1546     } elsif ($drive->{interface} eq 'usb') {
1547         die "implement me";
1548         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1549     } else {
1550         die "unsupported interface type";
1551     }
1552
1553     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1554
1555     if (my $serial = $drive->{serial}) {
1556         $serial = URI::Escape::uri_unescape($serial);
1557         $device .= ",serial=$serial";
1558     }
1559
1560
1561     return $device;
1562 }
1563
1564 sub get_initiator_name {
1565     my $initiator;
1566
1567     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1568     while (defined(my $line = <$fh>)) {
1569         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1570         $initiator = $1;
1571         last;
1572     }
1573     $fh->close();
1574
1575     return $initiator;
1576 }
1577
1578 my sub storage_allows_io_uring_default {
1579     my ($scfg, $cache_direct) = @_;
1580
1581     # io_uring with cache mode writeback or writethrough on krbd will hang...
1582     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1583
1584     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1585     # sometimes, just plain disable...
1586     return if $scfg && $scfg->{type} eq 'lvm';
1587
1588     # io_uring causes problems when used with CIFS since kernel 5.15
1589     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1590     return if $scfg && $scfg->{type} eq 'cifs';
1591
1592     return 1;
1593 }
1594
1595 my sub drive_uses_cache_direct {
1596     my ($drive, $scfg) = @_;
1597
1598     my $cache_direct = 0;
1599
1600     if (my $cache = $drive->{cache}) {
1601         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1602     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1603         $cache_direct = 1;
1604     }
1605
1606     return $cache_direct;
1607 }
1608
1609 sub print_drive_commandline_full {
1610     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1611
1612     my $path;
1613     my $volid = $drive->{file};
1614     my $format = $drive->{format};
1615     my $drive_id = get_drive_id($drive);
1616
1617     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1618     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1619
1620     if (drive_is_cdrom($drive)) {
1621         $path = get_iso_path($storecfg, $vmid, $volid);
1622         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1623     } else {
1624         if ($storeid) {
1625             $path = PVE::Storage::path($storecfg, $volid);
1626             $format //= qemu_img_format($scfg, $volname);
1627         } else {
1628             $path = $volid;
1629             $format //= "raw";
1630         }
1631    }
1632
1633    my $is_rbd = $path =~ m/^rbd:/;
1634
1635     my $opts = '';
1636     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1637     foreach my $o (@qemu_drive_options) {
1638         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1639     }
1640
1641     # snapshot only accepts on|off
1642     if (defined($drive->{snapshot})) {
1643         my $v = $drive->{snapshot} ? 'on' : 'off';
1644         $opts .= ",snapshot=$v";
1645     }
1646
1647     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1648         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1649     }
1650
1651     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1652         my ($dir, $qmpname) = @$type;
1653         if (my $v = $drive->{"mbps$dir"}) {
1654             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1655         }
1656         if (my $v = $drive->{"mbps${dir}_max"}) {
1657             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1658         }
1659         if (my $v = $drive->{"bps${dir}_max_length"}) {
1660             $opts .= ",throttling.bps$qmpname-max-length=$v";
1661         }
1662         if (my $v = $drive->{"iops${dir}"}) {
1663             $opts .= ",throttling.iops$qmpname=$v";
1664         }
1665         if (my $v = $drive->{"iops${dir}_max"}) {
1666             $opts .= ",throttling.iops$qmpname-max=$v";
1667         }
1668         if (my $v = $drive->{"iops${dir}_max_length"}) {
1669             $opts .= ",throttling.iops$qmpname-max-length=$v";
1670         }
1671     }
1672
1673     if ($pbs_name) {
1674         $format = "rbd" if $is_rbd;
1675         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1676             if !$format;
1677         $opts .= ",format=alloc-track,file.driver=$format";
1678     } elsif ($format) {
1679         $opts .= ",format=$format";
1680     }
1681
1682     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1683
1684     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1685
1686     if (!$drive->{aio}) {
1687         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1688             # io_uring supports all cache modes
1689             $opts .= ",aio=io_uring";
1690         } else {
1691             # aio native works only with O_DIRECT
1692             if($cache_direct) {
1693                 $opts .= ",aio=native";
1694             } else {
1695                 $opts .= ",aio=threads";
1696             }
1697         }
1698     }
1699
1700     if (!drive_is_cdrom($drive)) {
1701         my $detectzeroes;
1702         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1703             $detectzeroes = 'off';
1704         } elsif ($drive->{discard}) {
1705             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1706         } else {
1707             # This used to be our default with discard not being specified:
1708             $detectzeroes = 'on';
1709         }
1710
1711         # note: 'detect-zeroes' works per blockdev and we want it to persist
1712         # after the alloc-track is removed, so put it on 'file' directly
1713         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1714         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1715     }
1716
1717     if ($pbs_name) {
1718         $opts .= ",backing=$pbs_name";
1719         $opts .= ",auto-remove=on";
1720     }
1721
1722     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1723     my $file_param = "file";
1724     if ($pbs_name) {
1725         # non-rbd drivers require the underlying file to be a seperate block
1726         # node, so add a second .file indirection
1727         $file_param .= ".file" if !$is_rbd;
1728         $file_param .= ".filename";
1729     }
1730     my $pathinfo = $path ? "$file_param=$path," : '';
1731
1732     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1733 }
1734
1735 sub print_pbs_blockdev {
1736     my ($pbs_conf, $pbs_name) = @_;
1737     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1738     $blockdev .= ",repository=$pbs_conf->{repository}";
1739     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1740     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1741     $blockdev .= ",archive=$pbs_conf->{archive}";
1742     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1743     return $blockdev;
1744 }
1745
1746 sub print_netdevice_full {
1747     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1748
1749     my $device = $net->{model};
1750     if ($net->{model} eq 'virtio') {
1751          $device = 'virtio-net-pci';
1752      };
1753
1754     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1755     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1756     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1757         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1758         # and out of each queue plus one config interrupt and control vector queue
1759         my $vectors = $net->{queues} * 2 + 2;
1760         $tmpstr .= ",vectors=$vectors,mq=on";
1761         if (min_version($machine_version, 7, 1)) {
1762             $tmpstr .= ",packed=on";
1763         }
1764     }
1765
1766     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1767         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1768     }
1769
1770     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1771
1772     if (my $mtu = $net->{mtu}) {
1773         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1774             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1775             if ($mtu == 1) {
1776                  $mtu = $bridge_mtu;
1777             } elsif ($mtu < 576) {
1778                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1779             } elsif ($mtu > $bridge_mtu) {
1780                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1781             }
1782             $tmpstr .= ",host_mtu=$mtu";
1783         } else {
1784             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1785         }
1786     }
1787
1788     if ($use_old_bios_files) {
1789         my $romfile;
1790         if ($device eq 'virtio-net-pci') {
1791             $romfile = 'pxe-virtio.rom';
1792         } elsif ($device eq 'e1000') {
1793             $romfile = 'pxe-e1000.rom';
1794         } elsif ($device eq 'e1000e') {
1795             $romfile = 'pxe-e1000e.rom';
1796         } elsif ($device eq 'ne2k') {
1797             $romfile = 'pxe-ne2k_pci.rom';
1798         } elsif ($device eq 'pcnet') {
1799             $romfile = 'pxe-pcnet.rom';
1800         } elsif ($device eq 'rtl8139') {
1801             $romfile = 'pxe-rtl8139.rom';
1802         }
1803         $tmpstr .= ",romfile=$romfile" if $romfile;
1804     }
1805
1806     return $tmpstr;
1807 }
1808
1809 sub print_netdev_full {
1810     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1811
1812     my $i = '';
1813     if ($netid =~ m/^net(\d+)$/) {
1814         $i = int($1);
1815     }
1816
1817     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1818
1819     my $ifname = "tap${vmid}i$i";
1820
1821     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1822     die "interface name '$ifname' is too long (max 15 character)\n"
1823         if length($ifname) >= 16;
1824
1825     my $vhostparam = '';
1826     if (is_native($arch)) {
1827         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1828     }
1829
1830     my $vmname = $conf->{name} || "vm$vmid";
1831
1832     my $netdev = "";
1833     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1834
1835     if ($net->{bridge}) {
1836         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1837             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1838     } else {
1839         $netdev = "type=user,id=$netid,hostname=$vmname";
1840     }
1841
1842     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1843
1844     return $netdev;
1845 }
1846
1847 my $vga_map = {
1848     'cirrus' => 'cirrus-vga',
1849     'std' => 'VGA',
1850     'vmware' => 'vmware-svga',
1851     'virtio' => 'virtio-vga',
1852     'virtio-gl' => 'virtio-vga-gl',
1853 };
1854
1855 sub print_vga_device {
1856     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1857
1858     my $type = $vga_map->{$vga->{type}};
1859     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1860         $type = 'virtio-gpu';
1861     }
1862     my $vgamem_mb = $vga->{memory};
1863
1864     my $max_outputs = '';
1865     if ($qxlnum) {
1866         $type = $id ? 'qxl' : 'qxl-vga';
1867
1868         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1869             # set max outputs so linux can have up to 4 qxl displays with one device
1870             if (min_version($machine_version, 4, 1)) {
1871                 $max_outputs = ",max_outputs=4";
1872             }
1873         }
1874     }
1875
1876     die "no devicetype for $vga->{type}\n" if !$type;
1877
1878     my $memory = "";
1879     if ($vgamem_mb) {
1880         if ($vga->{type} =~ /^virtio/) {
1881             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1882             $memory = ",max_hostmem=$bytes";
1883         } elsif ($qxlnum) {
1884             # from https://www.spice-space.org/multiple-monitors.html
1885             $memory = ",vgamem_mb=$vga->{memory}";
1886             my $ram = $vgamem_mb * 4;
1887             my $vram = $vgamem_mb * 2;
1888             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1889         } else {
1890             $memory = ",vgamem_mb=$vga->{memory}";
1891         }
1892     } elsif ($qxlnum && $id) {
1893         $memory = ",ram_size=67108864,vram_size=33554432";
1894     }
1895
1896     my $edidoff = "";
1897     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1898         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1899     }
1900
1901     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1902     my $vgaid = "vga" . ($id // '');
1903     my $pciaddr;
1904     if ($q35 && $vgaid eq 'vga') {
1905         # the first display uses pcie.0 bus on q35 machines
1906         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1907     } else {
1908         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1909     }
1910
1911     if ($vga->{type} eq 'virtio-gl') {
1912         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1913         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1914             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1915
1916         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1917             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1918     }
1919
1920     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1921 }
1922
1923 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1924 sub parse_net {
1925     my ($data, $disable_mac_autogen) = @_;
1926
1927     my $res = eval { parse_property_string($net_fmt, $data) };
1928     if ($@) {
1929         warn $@;
1930         return;
1931     }
1932     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1933         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1934         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1935     }
1936     return $res;
1937 }
1938
1939 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1940 sub parse_ipconfig {
1941     my ($data) = @_;
1942
1943     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1944     if ($@) {
1945         warn $@;
1946         return;
1947     }
1948
1949     if ($res->{gw} && !$res->{ip}) {
1950         warn 'gateway specified without specifying an IP address';
1951         return;
1952     }
1953     if ($res->{gw6} && !$res->{ip6}) {
1954         warn 'IPv6 gateway specified without specifying an IPv6 address';
1955         return;
1956     }
1957     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1958         warn 'gateway specified together with DHCP';
1959         return;
1960     }
1961     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1962         # gw6 + auto/dhcp
1963         warn "IPv6 gateway specified together with $res->{ip6} address";
1964         return;
1965     }
1966
1967     if (!$res->{ip} && !$res->{ip6}) {
1968         return { ip => 'dhcp', ip6 => 'dhcp' };
1969     }
1970
1971     return $res;
1972 }
1973
1974 sub print_net {
1975     my $net = shift;
1976
1977     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1978 }
1979
1980 sub add_random_macs {
1981     my ($settings) = @_;
1982
1983     foreach my $opt (keys %$settings) {
1984         next if $opt !~ m/^net(\d+)$/;
1985         my $net = parse_net($settings->{$opt});
1986         next if !$net;
1987         $settings->{$opt} = print_net($net);
1988     }
1989 }
1990
1991 sub vm_is_volid_owner {
1992     my ($storecfg, $vmid, $volid) = @_;
1993
1994     if ($volid !~  m|^/|) {
1995         my ($path, $owner);
1996         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1997         if ($owner && ($owner == $vmid)) {
1998             return 1;
1999         }
2000     }
2001
2002     return;
2003 }
2004
2005 sub vmconfig_register_unused_drive {
2006     my ($storecfg, $vmid, $conf, $drive) = @_;
2007
2008     if (drive_is_cloudinit($drive)) {
2009         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2010         warn $@ if $@;
2011         delete $conf->{cloudinit};
2012     } elsif (!drive_is_cdrom($drive)) {
2013         my $volid = $drive->{file};
2014         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2015             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2016         }
2017     }
2018 }
2019
2020 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2021 my $smbios1_fmt = {
2022     uuid => {
2023         type => 'string',
2024         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2025         format_description => 'UUID',
2026         description => "Set SMBIOS1 UUID.",
2027         optional => 1,
2028     },
2029     version => {
2030         type => 'string',
2031         pattern => '[A-Za-z0-9+\/]+={0,2}',
2032         format_description => 'Base64 encoded string',
2033         description => "Set SMBIOS1 version.",
2034         optional => 1,
2035     },
2036     serial => {
2037         type => 'string',
2038         pattern => '[A-Za-z0-9+\/]+={0,2}',
2039         format_description => 'Base64 encoded string',
2040         description => "Set SMBIOS1 serial number.",
2041         optional => 1,
2042     },
2043     manufacturer => {
2044         type => 'string',
2045         pattern => '[A-Za-z0-9+\/]+={0,2}',
2046         format_description => 'Base64 encoded string',
2047         description => "Set SMBIOS1 manufacturer.",
2048         optional => 1,
2049     },
2050     product => {
2051         type => 'string',
2052         pattern => '[A-Za-z0-9+\/]+={0,2}',
2053         format_description => 'Base64 encoded string',
2054         description => "Set SMBIOS1 product ID.",
2055         optional => 1,
2056     },
2057     sku => {
2058         type => 'string',
2059         pattern => '[A-Za-z0-9+\/]+={0,2}',
2060         format_description => 'Base64 encoded string',
2061         description => "Set SMBIOS1 SKU string.",
2062         optional => 1,
2063     },
2064     family => {
2065         type => 'string',
2066         pattern => '[A-Za-z0-9+\/]+={0,2}',
2067         format_description => 'Base64 encoded string',
2068         description => "Set SMBIOS1 family string.",
2069         optional => 1,
2070     },
2071     base64 => {
2072         type => 'boolean',
2073         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2074         optional => 1,
2075     },
2076 };
2077
2078 sub parse_smbios1 {
2079     my ($data) = @_;
2080
2081     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2082     warn $@ if $@;
2083     return $res;
2084 }
2085
2086 sub print_smbios1 {
2087     my ($smbios1) = @_;
2088     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2089 }
2090
2091 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2092
2093 sub parse_watchdog {
2094     my ($value) = @_;
2095
2096     return if !$value;
2097
2098     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2099     warn $@ if $@;
2100     return $res;
2101 }
2102
2103 sub parse_guest_agent {
2104     my ($conf) = @_;
2105
2106     return {} if !defined($conf->{agent});
2107
2108     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2109     warn $@ if $@;
2110
2111     # if the agent is disabled ignore the other potentially set properties
2112     return {} if !$res->{enabled};
2113     return $res;
2114 }
2115
2116 sub get_qga_key {
2117     my ($conf, $key) = @_;
2118     return undef if !defined($conf->{agent});
2119
2120     my $agent = parse_guest_agent($conf);
2121     return $agent->{$key};
2122 }
2123
2124 sub parse_vga {
2125     my ($value) = @_;
2126
2127     return {} if !$value;
2128     my $res = eval { parse_property_string($vga_fmt, $value) };
2129     warn $@ if $@;
2130     return $res;
2131 }
2132
2133 sub parse_rng {
2134     my ($value) = @_;
2135
2136     return if !$value;
2137
2138     my $res = eval { parse_property_string($rng_fmt, $value) };
2139     warn $@ if $@;
2140     return $res;
2141 }
2142
2143 sub parse_meta_info {
2144     my ($value) = @_;
2145
2146     return if !$value;
2147
2148     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2149     warn $@ if $@;
2150     return $res;
2151 }
2152
2153 sub new_meta_info_string {
2154     my () = @_; # for now do not allow to override any value
2155
2156     return PVE::JSONSchema::print_property_string(
2157         {
2158             'creation-qemu' => kvm_user_version(),
2159             ctime => "". int(time()),
2160         },
2161         $meta_info_fmt
2162     );
2163 }
2164
2165 sub qemu_created_version_fixups {
2166     my ($conf, $forcemachine, $kvmver) = @_;
2167
2168     my $meta = parse_meta_info($conf->{meta}) // {};
2169     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2170
2171     # check if we need to apply some handling for VMs that always use the latest machine version but
2172     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2173     # would be required (we do not want to pin machine version for non-windows OS type)
2174     if (
2175         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2176         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2177         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2178         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2179     ) {
2180         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2181         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2182             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2183             # and thus with the predictable interface naming of systemd
2184             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2185         }
2186     }
2187     return;
2188 }
2189
2190 # add JSON properties for create and set function
2191 sub json_config_properties {
2192     my ($prop, $with_disk_alloc) = @_;
2193
2194     my $skip_json_config_opts = {
2195         parent => 1,
2196         snaptime => 1,
2197         vmstate => 1,
2198         runningmachine => 1,
2199         runningcpu => 1,
2200         meta => 1,
2201     };
2202
2203     foreach my $opt (keys %$confdesc) {
2204         next if $skip_json_config_opts->{$opt};
2205
2206         if ($with_disk_alloc && is_valid_drivename($opt)) {
2207             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2208         } else {
2209             $prop->{$opt} = $confdesc->{$opt};
2210         }
2211     }
2212
2213     return $prop;
2214 }
2215
2216 # Properties that we can read from an OVF file
2217 sub json_ovf_properties {
2218     my $prop = {};
2219
2220     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2221         $prop->{$device} = {
2222             type => 'string',
2223             format => 'pve-volume-id-or-absolute-path',
2224             description => "Disk image that gets imported to $device",
2225             optional => 1,
2226         };
2227     }
2228
2229     $prop->{cores} = {
2230         type => 'integer',
2231         description => "The number of CPU cores.",
2232         optional => 1,
2233     };
2234     $prop->{memory} = {
2235         type => 'integer',
2236         description => "Amount of RAM for the VM in MB.",
2237         optional => 1,
2238     };
2239     $prop->{name} = {
2240         type => 'string',
2241         description => "Name of the VM.",
2242         optional => 1,
2243     };
2244
2245     return $prop;
2246 }
2247
2248 # return copy of $confdesc_cloudinit to generate documentation
2249 sub cloudinit_config_properties {
2250
2251     return dclone($confdesc_cloudinit);
2252 }
2253
2254 sub cloudinit_pending_properties {
2255     my $p = {
2256         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2257         name => 1,
2258     };
2259     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2260     return $p;
2261 }
2262
2263 sub check_type {
2264     my ($key, $value) = @_;
2265
2266     die "unknown setting '$key'\n" if !$confdesc->{$key};
2267
2268     my $type = $confdesc->{$key}->{type};
2269
2270     if (!defined($value)) {
2271         die "got undefined value\n";
2272     }
2273
2274     if ($value =~ m/[\n\r]/) {
2275         die "property contains a line feed\n";
2276     }
2277
2278     if ($type eq 'boolean') {
2279         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2280         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2281         die "type check ('boolean') failed - got '$value'\n";
2282     } elsif ($type eq 'integer') {
2283         return int($1) if $value =~ m/^(\d+)$/;
2284         die "type check ('integer') failed - got '$value'\n";
2285     } elsif ($type eq 'number') {
2286         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2287         die "type check ('number') failed - got '$value'\n";
2288     } elsif ($type eq 'string') {
2289         if (my $fmt = $confdesc->{$key}->{format}) {
2290             PVE::JSONSchema::check_format($fmt, $value);
2291             return $value;
2292         }
2293         $value =~ s/^\"(.*)\"$/$1/;
2294         return $value;
2295     } else {
2296         die "internal error"
2297     }
2298 }
2299
2300 sub destroy_vm {
2301     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2302
2303     my $conf = PVE::QemuConfig->load_config($vmid);
2304
2305     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2306         PVE::QemuConfig->check_lock($conf);
2307     }
2308
2309     if ($conf->{template}) {
2310         # check if any base image is still used by a linked clone
2311         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2312                 my ($ds, $drive) = @_;
2313                 return if drive_is_cdrom($drive);
2314
2315                 my $volid = $drive->{file};
2316                 return if !$volid || $volid =~ m|^/|;
2317
2318                 die "base volume '$volid' is still in use by linked cloned\n"
2319                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2320
2321         });
2322     }
2323
2324     my $volids = {};
2325     my $remove_owned_drive = sub {
2326         my ($ds, $drive) = @_;
2327         return if drive_is_cdrom($drive, 1);
2328
2329         my $volid = $drive->{file};
2330         return if !$volid || $volid =~ m|^/|;
2331         return if $volids->{$volid};
2332
2333         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2334         return if !$path || !$owner || ($owner != $vmid);
2335
2336         $volids->{$volid} = 1;
2337         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2338         warn "Could not remove disk '$volid', check manually: $@" if $@;
2339     };
2340
2341     # only remove disks owned by this VM (referenced in the config)
2342     my $include_opts = {
2343         include_unused => 1,
2344         extra_keys => ['vmstate'],
2345     };
2346     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2347
2348     for my $snap (values %{$conf->{snapshots}}) {
2349         next if !defined($snap->{vmstate});
2350         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2351         next if !defined($drive);
2352         $remove_owned_drive->('vmstate', $drive);
2353     }
2354
2355     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2356
2357     if ($purge_unreferenced) { # also remove unreferenced disk
2358         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2359         PVE::Storage::foreach_volid($vmdisks, sub {
2360             my ($volid, $sid, $volname, $d) = @_;
2361             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2362             warn $@ if $@;
2363         });
2364     }
2365
2366     eval { delete_ifaces_ipams_ips($conf, $vmid)};
2367     warn $@ if $@;
2368
2369     if (defined $replacement_conf) {
2370         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2371     } else {
2372         PVE::QemuConfig->destroy_config($vmid);
2373     }
2374 }
2375
2376 sub parse_vm_config {
2377     my ($filename, $raw, $strict) = @_;
2378
2379     return if !defined($raw);
2380
2381     my $res = {
2382         digest => Digest::SHA::sha1_hex($raw),
2383         snapshots => {},
2384         pending => {},
2385         cloudinit => {},
2386     };
2387
2388     my $handle_error = sub {
2389         my ($msg) = @_;
2390
2391         if ($strict) {
2392             die $msg;
2393         } else {
2394             warn $msg;
2395         }
2396     };
2397
2398     $filename =~ m|/qemu-server/(\d+)\.conf$|
2399         || die "got strange filename '$filename'";
2400
2401     my $vmid = $1;
2402
2403     my $conf = $res;
2404     my $descr;
2405     my $finish_description = sub {
2406         if (defined($descr)) {
2407             $descr =~ s/\s+$//;
2408             $conf->{description} = $descr;
2409         }
2410         $descr = undef;
2411     };
2412     my $section = '';
2413
2414     my @lines = split(/\n/, $raw);
2415     foreach my $line (@lines) {
2416         next if $line =~ m/^\s*$/;
2417
2418         if ($line =~ m/^\[PENDING\]\s*$/i) {
2419             $section = 'pending';
2420             $finish_description->();
2421             $conf = $res->{$section} = {};
2422             next;
2423         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2424             $section = 'cloudinit';
2425             $finish_description->();
2426             $conf = $res->{$section} = {};
2427             next;
2428
2429         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2430             $section = $1;
2431             $finish_description->();
2432             $conf = $res->{snapshots}->{$section} = {};
2433             next;
2434         }
2435
2436         if ($line =~ m/^\#(.*)$/) {
2437             $descr = '' if !defined($descr);
2438             $descr .= PVE::Tools::decode_text($1) . "\n";
2439             next;
2440         }
2441
2442         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2443             $descr = '' if !defined($descr);
2444             $descr .= PVE::Tools::decode_text($2);
2445         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2446             $conf->{snapstate} = $1;
2447         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2448             my $key = $1;
2449             my $value = $2;
2450             $conf->{$key} = $value;
2451         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2452             my $value = $1;
2453             if ($section eq 'pending') {
2454                 $conf->{delete} = $value; # we parse this later
2455             } else {
2456                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2457             }
2458         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2459             my $key = $1;
2460             my $value = $2;
2461             if ($section eq 'cloudinit') {
2462                 # ignore validation only used for informative purpose
2463                 $conf->{$key} = $value;
2464                 next;
2465             }
2466             eval { $value = check_type($key, $value); };
2467             if ($@) {
2468                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2469             } else {
2470                 $key = 'ide2' if $key eq 'cdrom';
2471                 my $fmt = $confdesc->{$key}->{format};
2472                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2473                     my $v = parse_drive($key, $value);
2474                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2475                         $v->{file} = $volid;
2476                         $value = print_drive($v);
2477                     } else {
2478                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2479                         next;
2480                     }
2481                 }
2482
2483                 $conf->{$key} = $value;
2484             }
2485         } else {
2486             $handle_error->("vm $vmid - unable to parse config: $line\n");
2487         }
2488     }
2489
2490     $finish_description->();
2491     delete $res->{snapstate}; # just to be sure
2492
2493     return $res;
2494 }
2495
2496 sub write_vm_config {
2497     my ($filename, $conf) = @_;
2498
2499     delete $conf->{snapstate}; # just to be sure
2500
2501     if ($conf->{cdrom}) {
2502         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2503         $conf->{ide2} = $conf->{cdrom};
2504         delete $conf->{cdrom};
2505     }
2506
2507     # we do not use 'smp' any longer
2508     if ($conf->{sockets}) {
2509         delete $conf->{smp};
2510     } elsif ($conf->{smp}) {
2511         $conf->{sockets} = $conf->{smp};
2512         delete $conf->{cores};
2513         delete $conf->{smp};
2514     }
2515
2516     my $used_volids = {};
2517
2518     my $cleanup_config = sub {
2519         my ($cref, $pending, $snapname) = @_;
2520
2521         foreach my $key (keys %$cref) {
2522             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2523                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2524             my $value = $cref->{$key};
2525             if ($key eq 'delete') {
2526                 die "propertry 'delete' is only allowed in [PENDING]\n"
2527                     if !$pending;
2528                 # fixme: check syntax?
2529                 next;
2530             }
2531             eval { $value = check_type($key, $value); };
2532             die "unable to parse value of '$key' - $@" if $@;
2533
2534             $cref->{$key} = $value;
2535
2536             if (!$snapname && is_valid_drivename($key)) {
2537                 my $drive = parse_drive($key, $value);
2538                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2539             }
2540         }
2541     };
2542
2543     &$cleanup_config($conf);
2544
2545     &$cleanup_config($conf->{pending}, 1);
2546
2547     foreach my $snapname (keys %{$conf->{snapshots}}) {
2548         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2549         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2550     }
2551
2552     # remove 'unusedX' settings if we re-add a volume
2553     foreach my $key (keys %$conf) {
2554         my $value = $conf->{$key};
2555         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2556             delete $conf->{$key};
2557         }
2558     }
2559
2560     my $generate_raw_config = sub {
2561         my ($conf, $pending) = @_;
2562
2563         my $raw = '';
2564
2565         # add description as comment to top of file
2566         if (defined(my $descr = $conf->{description})) {
2567             if ($descr) {
2568                 foreach my $cl (split(/\n/, $descr)) {
2569                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2570                 }
2571             } else {
2572                 $raw .= "#\n" if $pending;
2573             }
2574         }
2575
2576         foreach my $key (sort keys %$conf) {
2577             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2578             $raw .= "$key: $conf->{$key}\n";
2579         }
2580         return $raw;
2581     };
2582
2583     my $raw = &$generate_raw_config($conf);
2584
2585     if (scalar(keys %{$conf->{pending}})){
2586         $raw .= "\n[PENDING]\n";
2587         $raw .= &$generate_raw_config($conf->{pending}, 1);
2588     }
2589
2590     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2591         $raw .= "\n[special:cloudinit]\n";
2592         $raw .= &$generate_raw_config($conf->{cloudinit});
2593     }
2594
2595     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2596         $raw .= "\n[$snapname]\n";
2597         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2598     }
2599
2600     return $raw;
2601 }
2602
2603 sub load_defaults {
2604
2605     my $res = {};
2606
2607     # we use static defaults from our JSON schema configuration
2608     foreach my $key (keys %$confdesc) {
2609         if (defined(my $default = $confdesc->{$key}->{default})) {
2610             $res->{$key} = $default;
2611         }
2612     }
2613
2614     return $res;
2615 }
2616
2617 sub config_list {
2618     my $vmlist = PVE::Cluster::get_vmlist();
2619     my $res = {};
2620     return $res if !$vmlist || !$vmlist->{ids};
2621     my $ids = $vmlist->{ids};
2622     my $nodename = nodename();
2623
2624     foreach my $vmid (keys %$ids) {
2625         my $d = $ids->{$vmid};
2626         next if !$d->{node} || $d->{node} ne $nodename;
2627         next if !$d->{type} || $d->{type} ne 'qemu';
2628         $res->{$vmid}->{exists} = 1;
2629     }
2630     return $res;
2631 }
2632
2633 # test if VM uses local resources (to prevent migration)
2634 sub check_local_resources {
2635     my ($conf, $noerr) = @_;
2636
2637     my @loc_res = ();
2638     my $mapped_res = [];
2639
2640     my $nodelist = PVE::Cluster::get_nodelist();
2641     my $pci_map = PVE::Mapping::PCI::config();
2642     my $usb_map = PVE::Mapping::USB::config();
2643
2644     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2645
2646     my $add_missing_mapping = sub {
2647         my ($type, $key, $id) = @_;
2648         for my $node (@$nodelist) {
2649             my $entry;
2650             if ($type eq 'pci') {
2651                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2652             } elsif ($type eq 'usb') {
2653                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2654             }
2655             if (!scalar($entry->@*)) {
2656                 push @{$missing_mappings_by_node->{$node}}, $key;
2657             }
2658         }
2659     };
2660
2661     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2662     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2663
2664     push @loc_res, "ivshmem" if $conf->{ivshmem};
2665
2666     foreach my $k (keys %$conf) {
2667         if ($k =~ m/^usb/) {
2668             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2669             next if $entry->{host} =~ m/^spice$/i;
2670             if ($entry->{mapping}) {
2671                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2672                 push @$mapped_res, $k;
2673             }
2674         }
2675         if ($k =~ m/^hostpci/) {
2676             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2677             if ($entry->{mapping}) {
2678                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2679                 push @$mapped_res, $k;
2680             }
2681         }
2682         # sockets are safe: they will recreated be on the target side post-migrate
2683         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2684         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2685     }
2686
2687     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2688
2689     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2690 }
2691
2692 # check if used storages are available on all nodes (use by migrate)
2693 sub check_storage_availability {
2694     my ($storecfg, $conf, $node) = @_;
2695
2696     PVE::QemuConfig->foreach_volume($conf, sub {
2697         my ($ds, $drive) = @_;
2698
2699         my $volid = $drive->{file};
2700         return if !$volid;
2701
2702         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2703         return if !$sid;
2704
2705         # check if storage is available on both nodes
2706         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2707         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2708
2709         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2710
2711         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2712             if !$scfg->{content}->{$vtype};
2713    });
2714 }
2715
2716 # list nodes where all VM images are available (used by has_feature API)
2717 sub shared_nodes {
2718     my ($conf, $storecfg) = @_;
2719
2720     my $nodelist = PVE::Cluster::get_nodelist();
2721     my $nodehash = { map { $_ => 1 } @$nodelist };
2722     my $nodename = nodename();
2723
2724     PVE::QemuConfig->foreach_volume($conf, sub {
2725         my ($ds, $drive) = @_;
2726
2727         my $volid = $drive->{file};
2728         return if !$volid;
2729
2730         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2731         if ($storeid) {
2732             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2733             if ($scfg->{disable}) {
2734                 $nodehash = {};
2735             } elsif (my $avail = $scfg->{nodes}) {
2736                 foreach my $node (keys %$nodehash) {
2737                     delete $nodehash->{$node} if !$avail->{$node};
2738                 }
2739             } elsif (!$scfg->{shared}) {
2740                 foreach my $node (keys %$nodehash) {
2741                     delete $nodehash->{$node} if $node ne $nodename
2742                 }
2743             }
2744         }
2745     });
2746
2747     return $nodehash
2748 }
2749
2750 sub check_local_storage_availability {
2751     my ($conf, $storecfg) = @_;
2752
2753     my $nodelist = PVE::Cluster::get_nodelist();
2754     my $nodehash = { map { $_ => {} } @$nodelist };
2755
2756     PVE::QemuConfig->foreach_volume($conf, sub {
2757         my ($ds, $drive) = @_;
2758
2759         my $volid = $drive->{file};
2760         return if !$volid;
2761
2762         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2763         if ($storeid) {
2764             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2765
2766             if ($scfg->{disable}) {
2767                 foreach my $node (keys %$nodehash) {
2768                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2769                 }
2770             } elsif (my $avail = $scfg->{nodes}) {
2771                 foreach my $node (keys %$nodehash) {
2772                     if (!$avail->{$node}) {
2773                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2774                     }
2775                 }
2776             }
2777         }
2778     });
2779
2780     foreach my $node (values %$nodehash) {
2781         if (my $unavail = $node->{unavailable_storages}) {
2782             $node->{unavailable_storages} = [ sort keys %$unavail ];
2783         }
2784     }
2785
2786     return $nodehash
2787 }
2788
2789 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2790 sub check_running {
2791     my ($vmid, $nocheck, $node) = @_;
2792
2793     # $nocheck is set when called during a migration, in which case the config
2794     # file might still or already reside on the *other* node
2795     # - because rename has already happened, and current node is source
2796     # - because rename hasn't happened yet, and current node is target
2797     # - because rename has happened, current node is target, but hasn't yet
2798     # processed it yet
2799     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2800     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2801 }
2802
2803 sub vzlist {
2804
2805     my $vzlist = config_list();
2806
2807     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2808
2809     while (defined(my $de = $fd->read)) {
2810         next if $de !~ m/^(\d+)\.pid$/;
2811         my $vmid = $1;
2812         next if !defined($vzlist->{$vmid});
2813         if (my $pid = check_running($vmid)) {
2814             $vzlist->{$vmid}->{pid} = $pid;
2815         }
2816     }
2817
2818     return $vzlist;
2819 }
2820
2821 our $vmstatus_return_properties = {
2822     vmid => get_standard_option('pve-vmid'),
2823     status => {
2824         description => "QEMU process status.",
2825         type => 'string',
2826         enum => ['stopped', 'running'],
2827     },
2828     maxmem => {
2829         description => "Maximum memory in bytes.",
2830         type => 'integer',
2831         optional => 1,
2832         renderer => 'bytes',
2833     },
2834     maxdisk => {
2835         description => "Root disk size in bytes.",
2836         type => 'integer',
2837         optional => 1,
2838         renderer => 'bytes',
2839     },
2840     name => {
2841         description => "VM name.",
2842         type => 'string',
2843         optional => 1,
2844     },
2845     qmpstatus => {
2846         description => "VM run state from the 'query-status' QMP monitor command.",
2847         type => 'string',
2848         optional => 1,
2849     },
2850     pid => {
2851         description => "PID of running qemu process.",
2852         type => 'integer',
2853         optional => 1,
2854     },
2855     uptime => {
2856         description => "Uptime.",
2857         type => 'integer',
2858         optional => 1,
2859         renderer => 'duration',
2860     },
2861     cpus => {
2862         description => "Maximum usable CPUs.",
2863         type => 'number',
2864         optional => 1,
2865     },
2866     lock => {
2867         description => "The current config lock, if any.",
2868         type => 'string',
2869         optional => 1,
2870     },
2871     tags => {
2872         description  => "The current configured tags, if any",
2873         type => 'string',
2874         optional => 1,
2875     },
2876     'running-machine' => {
2877         description => "The currently running machine type (if running).",
2878         type => 'string',
2879         optional => 1,
2880     },
2881     'running-qemu' => {
2882         description => "The currently running QEMU version (if running).",
2883         type => 'string',
2884         optional => 1,
2885     },
2886 };
2887
2888 my $last_proc_pid_stat;
2889
2890 # get VM status information
2891 # This must be fast and should not block ($full == false)
2892 # We only query KVM using QMP if $full == true (this can be slow)
2893 sub vmstatus {
2894     my ($opt_vmid, $full) = @_;
2895
2896     my $res = {};
2897
2898     my $storecfg = PVE::Storage::config();
2899
2900     my $list = vzlist();
2901     my $defaults = load_defaults();
2902
2903     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2904
2905     my $cpucount = $cpuinfo->{cpus} || 1;
2906
2907     foreach my $vmid (keys %$list) {
2908         next if $opt_vmid && ($vmid ne $opt_vmid);
2909
2910         my $conf = PVE::QemuConfig->load_config($vmid);
2911
2912         my $d = { vmid => int($vmid) };
2913         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2914
2915         # fixme: better status?
2916         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2917
2918         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2919         if (defined($size)) {
2920             $d->{disk} = 0; # no info available
2921             $d->{maxdisk} = $size;
2922         } else {
2923             $d->{disk} = 0;
2924             $d->{maxdisk} = 0;
2925         }
2926
2927         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2928             * ($conf->{cores} || $defaults->{cores});
2929         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2930         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2931
2932         $d->{name} = $conf->{name} || "VM $vmid";
2933         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2934
2935         if ($conf->{balloon}) {
2936             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2937             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2938                 : $defaults->{shares};
2939         }
2940
2941         $d->{uptime} = 0;
2942         $d->{cpu} = 0;
2943         $d->{mem} = 0;
2944
2945         $d->{netout} = 0;
2946         $d->{netin} = 0;
2947
2948         $d->{diskread} = 0;
2949         $d->{diskwrite} = 0;
2950
2951         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2952
2953         $d->{serial} = 1 if conf_has_serial($conf);
2954         $d->{lock} = $conf->{lock} if $conf->{lock};
2955         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2956
2957         $res->{$vmid} = $d;
2958     }
2959
2960     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2961     foreach my $dev (keys %$netdev) {
2962         next if $dev !~ m/^tap([1-9]\d*)i/;
2963         my $vmid = $1;
2964         my $d = $res->{$vmid};
2965         next if !$d;
2966
2967         $d->{netout} += $netdev->{$dev}->{receive};
2968         $d->{netin} += $netdev->{$dev}->{transmit};
2969
2970         if ($full) {
2971             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2972             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2973         }
2974
2975     }
2976
2977     my $ctime = gettimeofday;
2978
2979     foreach my $vmid (keys %$list) {
2980
2981         my $d = $res->{$vmid};
2982         my $pid = $d->{pid};
2983         next if !$pid;
2984
2985         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2986         next if !$pstat; # not running
2987
2988         my $used = $pstat->{utime} + $pstat->{stime};
2989
2990         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2991
2992         if ($pstat->{vsize}) {
2993             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2994         }
2995
2996         my $old = $last_proc_pid_stat->{$pid};
2997         if (!$old) {
2998             $last_proc_pid_stat->{$pid} = {
2999                 time => $ctime,
3000                 used => $used,
3001                 cpu => 0,
3002             };
3003             next;
3004         }
3005
3006         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3007
3008         if ($dtime > 1000) {
3009             my $dutime = $used -  $old->{used};
3010
3011             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3012             $last_proc_pid_stat->{$pid} = {
3013                 time => $ctime,
3014                 used => $used,
3015                 cpu => $d->{cpu},
3016             };
3017         } else {
3018             $d->{cpu} = $old->{cpu};
3019         }
3020     }
3021
3022     return $res if !$full;
3023
3024     my $qmpclient = PVE::QMPClient->new();
3025
3026     my $ballooncb = sub {
3027         my ($vmid, $resp) = @_;
3028
3029         my $info = $resp->{'return'};
3030         return if !$info->{max_mem};
3031
3032         my $d = $res->{$vmid};
3033
3034         # use memory assigned to VM
3035         $d->{maxmem} = $info->{max_mem};
3036         $d->{balloon} = $info->{actual};
3037
3038         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3039             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3040             $d->{freemem} = $info->{free_mem};
3041         }
3042
3043         $d->{ballooninfo} = $info;
3044     };
3045
3046     my $blockstatscb = sub {
3047         my ($vmid, $resp) = @_;
3048         my $data = $resp->{'return'} || [];
3049         my $totalrdbytes = 0;
3050         my $totalwrbytes = 0;
3051
3052         for my $blockstat (@$data) {
3053             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3054             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3055
3056             $blockstat->{device} =~ s/drive-//;
3057             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3058         }
3059         $res->{$vmid}->{diskread} = $totalrdbytes;
3060         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3061     };
3062
3063     my $machinecb = sub {
3064         my ($vmid, $resp) = @_;
3065         my $data = $resp->{'return'} || [];
3066
3067         $res->{$vmid}->{'running-machine'} =
3068             PVE::QemuServer::Machine::current_from_query_machines($data);
3069     };
3070
3071     my $versioncb = sub {
3072         my ($vmid, $resp) = @_;
3073         my $data = $resp->{'return'} // {};
3074         my $version = 'unknown';
3075
3076         if (my $v = $data->{qemu}) {
3077             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3078         }
3079
3080         $res->{$vmid}->{'running-qemu'} = $version;
3081     };
3082
3083     my $statuscb = sub {
3084         my ($vmid, $resp) = @_;
3085
3086         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3087         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3088         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3089         # this fails if ballon driver is not loaded, so this must be
3090         # the last commnand (following command are aborted if this fails).
3091         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3092
3093         my $status = 'unknown';
3094         if (!defined($status = $resp->{'return'}->{status})) {
3095             warn "unable to get VM status\n";
3096             return;
3097         }
3098
3099         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3100     };
3101
3102     foreach my $vmid (keys %$list) {
3103         next if $opt_vmid && ($vmid ne $opt_vmid);
3104         next if !$res->{$vmid}->{pid}; # not running
3105         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3106     }
3107
3108     $qmpclient->queue_execute(undef, 2);
3109
3110     foreach my $vmid (keys %$list) {
3111         next if $opt_vmid && ($vmid ne $opt_vmid);
3112         next if !$res->{$vmid}->{pid}; #not running
3113
3114         # we can't use the $qmpclient since it might have already aborted on
3115         # 'query-balloon', but this might also fail for older versions...
3116         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3117         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3118     }
3119
3120     foreach my $vmid (keys %$list) {
3121         next if $opt_vmid && ($vmid ne $opt_vmid);
3122         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3123     }
3124
3125     return $res;
3126 }
3127
3128 sub conf_has_serial {
3129     my ($conf) = @_;
3130
3131     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3132         if ($conf->{"serial$i"}) {
3133             return 1;
3134         }
3135     }
3136
3137     return 0;
3138 }
3139
3140 sub conf_has_audio {
3141     my ($conf, $id) = @_;
3142
3143     $id //= 0;
3144     my $audio = $conf->{"audio$id"};
3145     return if !defined($audio);
3146
3147     my $audioproperties = parse_property_string($audio_fmt, $audio);
3148     my $audiodriver = $audioproperties->{driver} // 'spice';
3149
3150     return {
3151         dev => $audioproperties->{device},
3152         dev_id => "audiodev$id",
3153         backend => $audiodriver,
3154         backend_id => "$audiodriver-backend${id}",
3155     };
3156 }
3157
3158 sub audio_devs {
3159     my ($audio, $audiopciaddr, $machine_version) = @_;
3160
3161     my $devs = [];
3162
3163     my $id = $audio->{dev_id};
3164     my $audiodev = "";
3165     if (min_version($machine_version, 4, 2)) {
3166         $audiodev = ",audiodev=$audio->{backend_id}";
3167     }
3168
3169     if ($audio->{dev} eq 'AC97') {
3170         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3171     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3172         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3173         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3174         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3175     } else {
3176         die "unkown audio device '$audio->{dev}', implement me!";
3177     }
3178
3179     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3180
3181     return $devs;
3182 }
3183
3184 sub get_tpm_paths {
3185     my ($vmid) = @_;
3186     return {
3187         socket => "/var/run/qemu-server/$vmid.swtpm",
3188         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3189     };
3190 }
3191
3192 sub add_tpm_device {
3193     my ($vmid, $devices, $conf) = @_;
3194
3195     return if !$conf->{tpmstate0};
3196
3197     my $paths = get_tpm_paths($vmid);
3198
3199     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3200     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3201     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3202 }
3203
3204 sub start_swtpm {
3205     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3206
3207     return if !$tpmdrive;
3208
3209     my $state;
3210     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3211     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3212     if ($storeid) {
3213         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3214     } else {
3215         $state = $tpm->{file};
3216     }
3217
3218     my $paths = get_tpm_paths($vmid);
3219
3220     # during migration, we will get state from remote
3221     #
3222     if (!$migration) {
3223         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3224         my $setup_cmd = [
3225             "swtpm_setup",
3226             "--tpmstate",
3227             "file://$state",
3228             "--createek",
3229             "--create-ek-cert",
3230             "--create-platform-cert",
3231             "--lock-nvram",
3232             "--config",
3233             "/etc/swtpm_setup.conf", # do not use XDG configs
3234             "--runas",
3235             "0", # force creation as root, error if not possible
3236             "--not-overwrite", # ignore existing state, do not modify
3237         ];
3238
3239         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3240         # TPM 2.0 supports ECC crypto, use if possible
3241         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3242
3243         run_command($setup_cmd, outfunc => sub {
3244             print "swtpm_setup: $1\n";
3245         });
3246     }
3247
3248     # Used to distinguish different invocations in the log.
3249     my $log_prefix = "[id=" . int(time()) . "] ";
3250
3251     my $emulator_cmd = [
3252         "swtpm",
3253         "socket",
3254         "--tpmstate",
3255         "backend-uri=file://$state,mode=0600",
3256         "--ctrl",
3257         "type=unixio,path=$paths->{socket},mode=0600",
3258         "--pid",
3259         "file=$paths->{pid}",
3260         "--terminate", # terminate on QEMU disconnect
3261         "--daemon",
3262         "--log",
3263         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3264     ];
3265     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3266     run_command($emulator_cmd, outfunc => sub { print $1; });
3267
3268     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3269     while (! -e $paths->{pid}) {
3270         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3271         usleep(50_000);
3272     }
3273
3274     # return untainted PID of swtpm daemon so it can be killed on error
3275     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3276     return $1;
3277 }
3278
3279 sub vga_conf_has_spice {
3280     my ($vga) = @_;
3281
3282     my $vgaconf = parse_vga($vga);
3283     my $vgatype = $vgaconf->{type};
3284     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3285
3286     return $1 || 1;
3287 }
3288
3289 sub is_native($) {
3290     my ($arch) = @_;
3291     return get_host_arch() eq $arch;
3292 }
3293
3294 sub get_vm_arch {
3295     my ($conf) = @_;
3296     return $conf->{arch} // get_host_arch();
3297 }
3298
3299 my $default_machines = {
3300     x86_64 => 'pc',
3301     aarch64 => 'virt',
3302 };
3303
3304 sub get_installed_machine_version {
3305     my ($kvmversion) = @_;
3306     $kvmversion = kvm_user_version() if !defined($kvmversion);
3307     $kvmversion =~ m/^(\d+\.\d+)/;
3308     return $1;
3309 }
3310
3311 sub windows_get_pinned_machine_version {
3312     my ($machine, $base_version, $kvmversion) = @_;
3313
3314     my $pin_version = $base_version;
3315     if (!defined($base_version) ||
3316         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3317     ) {
3318         $pin_version = get_installed_machine_version($kvmversion);
3319     }
3320     if (!$machine || $machine eq 'pc') {
3321         $machine = "pc-i440fx-$pin_version";
3322     } elsif ($machine eq 'q35') {
3323         $machine = "pc-q35-$pin_version";
3324     } elsif ($machine eq 'virt') {
3325         $machine = "virt-$pin_version";
3326     } else {
3327         warn "unknown machine type '$machine', not touching that!\n";
3328     }
3329
3330     return $machine;
3331 }
3332
3333 sub get_vm_machine {
3334     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3335
3336     my $machine = $forcemachine || $conf->{machine};
3337
3338     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3339         $kvmversion //= kvm_user_version();
3340         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3341         # layout which confuses windows quite a bit and may result in various regressions..
3342         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3343         if (windows_version($conf->{ostype})) {
3344             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3345         }
3346         $arch //= 'x86_64';
3347         $machine ||= $default_machines->{$arch};
3348         if ($add_pve_version) {
3349             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3350             $machine .= "+pve$pvever";
3351         }
3352     }
3353
3354     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3355         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3356         $machine = $1 if $is_pxe;
3357
3358         # for version-pinned machines that do not include a pve-version (e.g.
3359         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3360         $machine .= '+pve0';
3361
3362         $machine .= '.pxe' if $is_pxe;
3363     }
3364
3365     return $machine;
3366 }
3367
3368 sub get_ovmf_files($$$) {
3369     my ($arch, $efidisk, $smm) = @_;
3370
3371     my $types = $OVMF->{$arch}
3372         or die "no OVMF images known for architecture '$arch'\n";
3373
3374     my $type = 'default';
3375     if ($arch eq 'x86_64') {
3376         if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3377             $type = $smm ? "4m" : "4m-no-smm";
3378             $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3379         } else {
3380             # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3381         }
3382     }
3383
3384     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3385     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3386     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3387
3388     return ($ovmf_code, $ovmf_vars);
3389 }
3390
3391 my $Arch2Qemu = {
3392     aarch64 => '/usr/bin/qemu-system-aarch64',
3393     x86_64 => '/usr/bin/qemu-system-x86_64',
3394 };
3395 sub get_command_for_arch($) {
3396     my ($arch) = @_;
3397     return '/usr/bin/kvm' if is_native($arch);
3398
3399     my $cmd = $Arch2Qemu->{$arch}
3400         or die "don't know how to emulate architecture '$arch'\n";
3401     return $cmd;
3402 }
3403
3404 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3405 # to use in a QEMU command line (-cpu element), first array_intersect the result
3406 # of query_supported_ with query_understood_. This is necessary because:
3407 #
3408 # a) query_understood_ returns flags the host cannot use and
3409 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3410 #    flags, but CPU settings - with most of them being flags. Those settings
3411 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3412 #
3413 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3414 # expensive. If you need the value returned from this, you can get it much
3415 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3416 # $accel being 'kvm' or 'tcg'.
3417 #
3418 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3419 # changes, automatically populating pmxcfs.
3420 #
3421 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3422 # since kvm and tcg machines support different flags
3423 #
3424 sub query_supported_cpu_flags {
3425     my ($arch) = @_;
3426
3427     $arch //= get_host_arch();
3428     my $default_machine = $default_machines->{$arch};
3429
3430     my $flags = {};
3431
3432     # FIXME: Once this is merged, the code below should work for ARM as well:
3433     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3434     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3435         $arch eq "aarch64";
3436
3437     my $kvm_supported = defined(kvm_version());
3438     my $qemu_cmd = get_command_for_arch($arch);
3439     my $fakevmid = -1;
3440     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3441
3442     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3443     my $query_supported_run_qemu = sub {
3444         my ($kvm) = @_;
3445
3446         my $flags = {};
3447         my $cmd = [
3448             $qemu_cmd,
3449             '-machine', $default_machine,
3450             '-display', 'none',
3451             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3452             '-mon', 'chardev=qmp,mode=control',
3453             '-pidfile', $pidfile,
3454             '-S', '-daemonize'
3455         ];
3456
3457         if (!$kvm) {
3458             push @$cmd, '-accel', 'tcg';
3459         }
3460
3461         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3462         die "QEMU flag querying VM exited with code " . $rc if $rc;
3463
3464         eval {
3465             my $cmd_result = mon_cmd(
3466                 $fakevmid,
3467                 'query-cpu-model-expansion',
3468                 type => 'full',
3469                 model => { name => 'host' }
3470             );
3471
3472             my $props = $cmd_result->{model}->{props};
3473             foreach my $prop (keys %$props) {
3474                 next if $props->{$prop} ne '1';
3475                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3476                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3477                 # We only keep those with underscores, to match /proc/cpuinfo
3478                 $prop =~ s/\.|-/_/g;
3479                 $flags->{$prop} = 1;
3480             }
3481         };
3482         my $err = $@;
3483
3484         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3485         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3486
3487         die $err if $err;
3488
3489         return [ sort keys %$flags ];
3490     };
3491
3492     # We need to query QEMU twice, since KVM and TCG have different supported flags
3493     PVE::QemuConfig->lock_config($fakevmid, sub {
3494         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3495         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3496
3497         if ($kvm_supported) {
3498             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3499             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3500         }
3501     });
3502
3503     return $flags;
3504 }
3505
3506 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3507 my $understood_cpu_flag_dir = "/usr/share/kvm";
3508 sub query_understood_cpu_flags {
3509     my $arch = get_host_arch();
3510     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3511
3512     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3513         if ! -e $filepath;
3514
3515     my $raw = file_get_contents($filepath);
3516     $raw =~ s/^\s+|\s+$//g;
3517     my @flags = split(/\s+/, $raw);
3518
3519     return \@flags;
3520 }
3521
3522 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3523 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3524 my sub should_disable_smm {
3525     my ($conf, $vga, $machine) = @_;
3526
3527     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3528
3529     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3530         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3531 }
3532
3533 my sub print_ovmf_drive_commandlines {
3534     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3535
3536     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3537
3538     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3539
3540     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3541     if ($d) {
3542         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3543         my ($path, $format) = $d->@{'file', 'format'};
3544         if ($storeid) {
3545             $path = PVE::Storage::path($storecfg, $d->{file});
3546             if (!defined($format)) {
3547                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3548                 $format = qemu_img_format($scfg, $volname);
3549             }
3550         } elsif (!defined($format)) {
3551             die "efidisk format must be specified\n";
3552         }
3553         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3554         if ($path =~ m/^rbd:/) {
3555             $var_drive_str .= ',cache=writeback';
3556             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3557         }
3558         $var_drive_str .= ",format=$format,file=$path";
3559
3560         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3561         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3562     } else {
3563         log_warn("no efidisk configured! Using temporary efivars disk.");
3564         my $path = "/tmp/$vmid-ovmf.fd";
3565         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3566         $var_drive_str .= ",format=raw,file=$path";
3567         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3568     }
3569
3570     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3571 }
3572
3573 sub config_to_command {
3574     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3575         $pbs_backing) = @_;
3576
3577     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3578     my $devices = [];
3579     my $bridges = {};
3580     my $ostype = $conf->{ostype};
3581     my $winversion = windows_version($ostype);
3582     my $kvm = $conf->{kvm};
3583     my $nodename = nodename();
3584
3585     my $arch = get_vm_arch($conf);
3586     my $kvm_binary = get_command_for_arch($arch);
3587     my $kvmver = kvm_user_version($kvm_binary);
3588
3589     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3590         $kvmver //= "undefined";
3591         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3592     }
3593
3594     my $add_pve_version = min_version($kvmver, 4, 1);
3595
3596     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3597     my $machine_version = extract_version($machine_type, $kvmver);
3598     $kvm //= 1 if is_native($arch);
3599
3600     $machine_version =~ m/(\d+)\.(\d+)/;
3601     my ($machine_major, $machine_minor) = ($1, $2);
3602
3603     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3604         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3605     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3606         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3607             ." please upgrade node '$nodename'\n"
3608     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3609         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3610         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3611             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3612             ." node '$nodename'\n";
3613     }
3614
3615     # if a specific +pve version is required for a feature, use $version_guard
3616     # instead of min_version to allow machines to be run with the minimum
3617     # required version
3618     my $required_pve_version = 0;
3619     my $version_guard = sub {
3620         my ($major, $minor, $pve) = @_;
3621         return 0 if !min_version($machine_version, $major, $minor, $pve);
3622         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3623         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3624         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3625         return 1;
3626     };
3627
3628     if ($kvm && !defined kvm_version()) {
3629         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3630             ." or enable in BIOS.\n";
3631     }
3632
3633     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3634     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3635     my $use_old_bios_files = undef;
3636     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3637
3638     my $cmd = [];
3639     if ($conf->{affinity}) {
3640         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3641     }
3642
3643     push @$cmd, $kvm_binary;
3644
3645     push @$cmd, '-id', $vmid;
3646
3647     my $vmname = $conf->{name} || "vm$vmid";
3648
3649     push @$cmd, '-name', "$vmname,debug-threads=on";
3650
3651     push @$cmd, '-no-shutdown';
3652
3653     my $use_virtio = 0;
3654
3655     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3656     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3657     push @$cmd, '-mon', "chardev=qmp,mode=control";
3658
3659     if (min_version($machine_version, 2, 12)) {
3660         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3661         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3662     }
3663
3664     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3665
3666     push @$cmd, '-daemonize';
3667
3668     if ($conf->{smbios1}) {
3669         my $smbios_conf = parse_smbios1($conf->{smbios1});
3670         if ($smbios_conf->{base64}) {
3671             # Do not pass base64 flag to qemu
3672             delete $smbios_conf->{base64};
3673             my $smbios_string = "";
3674             foreach my $key (keys %$smbios_conf) {
3675                 my $value;
3676                 if ($key eq "uuid") {
3677                     $value = $smbios_conf->{uuid}
3678                 } else {
3679                     $value = decode_base64($smbios_conf->{$key});
3680                 }
3681                 # qemu accepts any binary data, only commas need escaping by double comma
3682                 $value =~ s/,/,,/g;
3683                 $smbios_string .= "," . $key . "=" . $value if $value;
3684             }
3685             push @$cmd, '-smbios', "type=1" . $smbios_string;
3686         } else {
3687             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3688         }
3689     }
3690
3691     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3692         my ($code_drive_str, $var_drive_str) =
3693             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3694         push $cmd->@*, '-drive', $code_drive_str;
3695         push $cmd->@*, '-drive', $var_drive_str;
3696     }
3697
3698     if ($q35) { # tell QEMU to load q35 config early
3699         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3700         if (min_version($machine_version, 4, 0)) {
3701             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3702         } else {
3703             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3704         }
3705     }
3706
3707     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3708         push @$cmd, $fixups->@*;
3709     }
3710
3711     if ($conf->{vmgenid}) {
3712         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3713     }
3714
3715     # add usb controllers
3716     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3717         $conf, $bridges, $arch, $machine_type, $machine_version);
3718     push @$devices, @usbcontrollers if @usbcontrollers;
3719     my $vga = parse_vga($conf->{vga});
3720
3721     my $qxlnum = vga_conf_has_spice($conf->{vga});
3722     $vga->{type} = 'qxl' if $qxlnum;
3723
3724     if (!$vga->{type}) {
3725         if ($arch eq 'aarch64') {
3726             $vga->{type} = 'virtio';
3727         } elsif (min_version($machine_version, 2, 9)) {
3728             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3729         } else {
3730             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3731         }
3732     }
3733
3734     # enable absolute mouse coordinates (needed by vnc)
3735     my $tablet = $conf->{tablet};
3736     if (!defined($tablet)) {
3737         $tablet = $defaults->{tablet};
3738         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3739         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3740     }
3741
3742     if ($tablet) {
3743         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3744         my $kbd = print_keyboarddevice_full($conf, $arch);
3745         push @$devices, '-device', $kbd if defined($kbd);
3746     }
3747
3748     my $bootorder = device_bootorder($conf);
3749
3750     # host pci device passthrough
3751     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3752         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3753
3754     # usb devices
3755     my $usb_dev_features = {};
3756     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3757
3758     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3759         $conf, $usb_dev_features, $bootorder, $machine_version);
3760     push @$devices, @usbdevices if @usbdevices;
3761
3762     # serial devices
3763     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3764         my $path = $conf->{"serial$i"} or next;
3765         if ($path eq 'socket') {
3766             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3767             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3768             # On aarch64, serial0 is the UART device. QEMU only allows
3769             # connecting UART devices via the '-serial' command line, as
3770             # the device has a fixed slot on the hardware...
3771             if ($arch eq 'aarch64' && $i == 0) {
3772                 push @$devices, '-serial', "chardev:serial$i";
3773             } else {
3774                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3775             }
3776         } else {
3777             die "no such serial device\n" if ! -c $path;
3778             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3779             push @$devices, '-device', "isa-serial,chardev=serial$i";
3780         }
3781     }
3782
3783     # parallel devices
3784     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3785         if (my $path = $conf->{"parallel$i"}) {
3786             die "no such parallel device\n" if ! -c $path;
3787             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3788             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3789             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3790         }
3791     }
3792
3793     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3794         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3795         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3796         push @$devices, @$audio_devs;
3797     }
3798
3799     # Add a TPM only if the VM is not a template,
3800     # to support backing up template VMs even if the TPM disk is write-protected.
3801     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3802
3803     my $sockets = 1;
3804     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3805     $sockets = $conf->{sockets} if  $conf->{sockets};
3806
3807     my $cores = $conf->{cores} || 1;
3808
3809     my $maxcpus = $sockets * $cores;
3810
3811     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3812
3813     my $allowed_vcpus = $cpuinfo->{cpus};
3814
3815     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3816
3817     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3818         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3819         for (my $i = 2; $i <= $vcpus; $i++)  {
3820             my $cpustr = print_cpu_device($conf,$i);
3821             push @$cmd, '-device', $cpustr;
3822         }
3823
3824     } else {
3825
3826         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3827     }
3828     push @$cmd, '-nodefaults';
3829
3830     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3831
3832     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3833
3834     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3835
3836     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3837         push @$devices, '-device', print_vga_device(
3838             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3839
3840         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3841
3842         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3843         push @$cmd,  '-vnc', "unix:$socket,password=on";
3844     } else {
3845         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3846         push @$cmd, '-nographic';
3847     }
3848
3849     # time drift fix
3850     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3851     my $useLocaltime = $conf->{localtime};
3852
3853     if ($winversion >= 5) { # windows
3854         $useLocaltime = 1 if !defined($conf->{localtime});
3855
3856         # use time drift fix when acpi is enabled
3857         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3858             $tdf = 1 if !defined($conf->{tdf});
3859         }
3860     }
3861
3862     if ($winversion >= 6) {
3863         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3864         push @$machineFlags, 'hpet=off';
3865     }
3866
3867     push @$rtcFlags, 'driftfix=slew' if $tdf;
3868
3869     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3870         push @$rtcFlags, "base=$conf->{startdate}";
3871     } elsif ($useLocaltime) {
3872         push @$rtcFlags, 'base=localtime';
3873     }
3874
3875     if ($forcecpu) {
3876         push @$cmd, '-cpu', $forcecpu;
3877     } else {
3878         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3879     }
3880
3881     PVE::QemuServer::Memory::config(
3882         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3883
3884     push @$cmd, '-S' if $conf->{freeze};
3885
3886     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3887
3888     my $guest_agent = parse_guest_agent($conf);
3889
3890     if ($guest_agent->{enabled}) {
3891         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3892         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3893
3894         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3895             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3896             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3897             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3898         } elsif ($guest_agent->{type} eq 'isa') {
3899             push @$devices, '-device', "isa-serial,chardev=qga0";
3900         }
3901     }
3902
3903     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3904     if ($rng && $version_guard->(4, 1, 2)) {
3905         check_rng_source($rng->{source});
3906
3907         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3908         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3909         my $limiter_str = "";
3910         if ($max_bytes) {
3911             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3912         }
3913
3914         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3915         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3916         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3917     }
3918
3919     my $spice_port;
3920
3921     assert_clipboard_config($vga);
3922     my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3923
3924     if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3925         if ($qxlnum > 1) {
3926             if ($winversion){
3927                 for (my $i = 1; $i < $qxlnum; $i++){
3928                     push @$devices, '-device', print_vga_device(
3929                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3930                 }
3931             } else {
3932                 # assume other OS works like Linux
3933                 my ($ram, $vram) = ("134217728", "67108864");
3934                 if ($vga->{memory}) {
3935                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3936                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3937                 }
3938                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3939                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3940             }
3941         }
3942
3943         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3944
3945         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3946         if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3947             push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3948         } else {
3949             push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3950         }
3951         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3952
3953         if ($is_spice) {
3954             my $pfamily = PVE::Tools::get_host_address_family($nodename);
3955             my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3956             die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3957
3958             my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3959             $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3960
3961             my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3962             my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3963             if ($spice_enhancement->{foldersharing}) {
3964                 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3965                 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3966             }
3967
3968             my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3969             $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3970                 if $spice_enhancement->{videostreaming};
3971             push @$devices, '-spice', "$spice_opts";
3972         }
3973     }
3974
3975     # enable balloon by default, unless explicitly disabled
3976     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3977         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3978         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3979         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3980         push @$devices, '-device', $ballooncmd;
3981     }
3982
3983     if ($conf->{watchdog}) {
3984         my $wdopts = parse_watchdog($conf->{watchdog});
3985         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3986         my $watchdog = $wdopts->{model} || 'i6300esb';
3987         push @$devices, '-device', "$watchdog$pciaddr";
3988         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3989     }
3990
3991     my $vollist = [];
3992     my $scsicontroller = {};
3993     my $ahcicontroller = {};
3994     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3995
3996     # Add iscsi initiator name if available
3997     if (my $initiator = get_initiator_name()) {
3998         push @$devices, '-iscsi', "initiator-name=$initiator";
3999     }
4000
4001     PVE::QemuConfig->foreach_volume($conf, sub {
4002         my ($ds, $drive) = @_;
4003
4004         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4005             check_volume_storage_type($storecfg, $drive->{file});
4006             push @$vollist, $drive->{file};
4007         }
4008
4009         # ignore efidisk here, already added in bios/fw handling code above
4010         return if $drive->{interface} eq 'efidisk';
4011         # similar for TPM
4012         return if $drive->{interface} eq 'tpmstate';
4013
4014         $use_virtio = 1 if $ds =~ m/^virtio/;
4015
4016         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4017
4018         if ($drive->{interface} eq 'virtio'){
4019            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4020         }
4021
4022         if ($drive->{interface} eq 'scsi') {
4023
4024             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4025
4026             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4027                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4028
4029             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4030             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4031
4032             my $iothread = '';
4033             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4034                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4035                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4036             } elsif ($drive->{iothread}) {
4037                 log_warn(
4038                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4039                 );
4040             }
4041
4042             my $queues = '';
4043             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4044                 $queues = ",num_queues=$drive->{queues}";
4045             }
4046
4047             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4048                 if !$scsicontroller->{$controller};
4049             $scsicontroller->{$controller}=1;
4050         }
4051
4052         if ($drive->{interface} eq 'sata') {
4053             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4054             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4055             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4056                 if !$ahcicontroller->{$controller};
4057             $ahcicontroller->{$controller}=1;
4058         }
4059
4060         my $pbs_conf = $pbs_backing->{$ds};
4061         my $pbs_name = undef;
4062         if ($pbs_conf) {
4063             $pbs_name = "drive-$ds-pbs";
4064             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4065         }
4066
4067         my $drive_cmd = print_drive_commandline_full(
4068             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4069
4070         # extra protection for templates, but SATA and IDE don't support it..
4071         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4072
4073         push @$devices, '-drive',$drive_cmd;
4074         push @$devices, '-device', print_drivedevice_full(
4075             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4076     });
4077
4078     for (my $i = 0; $i < $MAX_NETS; $i++) {
4079         my $netname = "net$i";
4080
4081         next if !$conf->{$netname};
4082         my $d = parse_net($conf->{$netname});
4083         next if !$d;
4084         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4085
4086         $use_virtio = 1 if $d->{model} eq 'virtio';
4087
4088         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4089
4090         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4091         push @$devices, '-netdev', $netdevfull;
4092
4093         my $netdevicefull = print_netdevice_full(
4094             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4095
4096         push @$devices, '-device', $netdevicefull;
4097     }
4098
4099     if ($conf->{ivshmem}) {
4100         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4101
4102         my $bus;
4103         if ($q35) {
4104             $bus = print_pcie_addr("ivshmem");
4105         } else {
4106             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4107         }
4108
4109         my $ivshmem_name = $ivshmem->{name} // $vmid;
4110         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4111
4112         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4113         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4114             .",size=$ivshmem->{size}M";
4115     }
4116
4117     # pci.4 is nested in pci.1
4118     $bridges->{1} = 1 if $bridges->{4};
4119
4120     if (!$q35) { # add pci bridges
4121         if (min_version($machine_version, 2, 3)) {
4122            $bridges->{1} = 1;
4123            $bridges->{2} = 1;
4124         }
4125         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4126     }
4127
4128     for my $k (sort {$b cmp $a} keys %$bridges) {
4129         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4130
4131         my $k_name = $k;
4132         if ($k == 2 && $legacy_igd) {
4133             $k_name = "$k-igd";
4134         }
4135         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4136         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4137
4138         if ($q35) { # add after -readconfig pve-q35.cfg
4139             splice @$devices, 2, 0, '-device', $devstr;
4140         } else {
4141             unshift @$devices, '-device', $devstr if $k > 0;
4142         }
4143     }
4144
4145     if (!$kvm) {
4146         push @$machineFlags, 'accel=tcg';
4147     }
4148
4149     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4150
4151     my $machine_type_min = $machine_type;
4152     if ($add_pve_version) {
4153         $machine_type_min =~ s/\+pve\d+$//;
4154         $machine_type_min .= "+pve$required_pve_version";
4155     }
4156     push @$machineFlags, "type=${machine_type_min}";
4157
4158     push @$cmd, @$devices;
4159     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4160     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4161     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4162
4163     if (my $vmstate = $conf->{vmstate}) {
4164         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4165         push @$vollist, $vmstate;
4166         push @$cmd, '-loadstate', $statepath;
4167         print "activating and using '$vmstate' as vmstate\n";
4168     }
4169
4170     if (PVE::QemuConfig->is_template($conf)) {
4171         # needed to workaround base volumes being read-only
4172         push @$cmd, '-snapshot';
4173     }
4174
4175     # add custom args
4176     if ($conf->{args}) {
4177         my $aa = PVE::Tools::split_args($conf->{args});
4178         push @$cmd, @$aa;
4179     }
4180
4181     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4182 }
4183
4184 sub check_rng_source {
4185     my ($source) = @_;
4186
4187     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4188     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4189         if ! -e $source;
4190
4191     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4192     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4193         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4194         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4195         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4196             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4197             ." to the host.\n";
4198     }
4199 }
4200
4201 sub spice_port {
4202     my ($vmid) = @_;
4203
4204     my $res = mon_cmd($vmid, 'query-spice');
4205
4206     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4207 }
4208
4209 sub vm_devices_list {
4210     my ($vmid) = @_;
4211
4212     my $res = mon_cmd($vmid, 'query-pci');
4213     my $devices_to_check = [];
4214     my $devices = {};
4215     foreach my $pcibus (@$res) {
4216         push @$devices_to_check, @{$pcibus->{devices}},
4217     }
4218
4219     while (@$devices_to_check) {
4220         my $to_check = [];
4221         for my $d (@$devices_to_check) {
4222             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4223             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4224
4225             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4226             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4227         }
4228         $devices_to_check = $to_check;
4229     }
4230
4231     my $resblock = mon_cmd($vmid, 'query-block');
4232     foreach my $block (@$resblock) {
4233         if($block->{device} =~ m/^drive-(\S+)/){
4234                 $devices->{$1} = 1;
4235         }
4236     }
4237
4238     my $resmice = mon_cmd($vmid, 'query-mice');
4239     foreach my $mice (@$resmice) {
4240         if ($mice->{name} eq 'QEMU HID Tablet') {
4241             $devices->{tablet} = 1;
4242             last;
4243         }
4244     }
4245
4246     # for usb devices there is no query-usb
4247     # but we can iterate over the entries in
4248     # qom-list path=/machine/peripheral
4249     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4250     foreach my $per (@$resperipheral) {
4251         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4252             $devices->{$per->{name}} = 1;
4253         }
4254     }
4255
4256     return $devices;
4257 }
4258
4259 sub vm_deviceplug {
4260     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4261
4262     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4263
4264     my $devices_list = vm_devices_list($vmid);
4265     return 1 if defined($devices_list->{$deviceid});
4266
4267     # add PCI bridge if we need it for the device
4268     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4269
4270     if ($deviceid eq 'tablet') {
4271         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4272     } elsif ($deviceid eq 'keyboard') {
4273         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4274     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4275         my $id = $1;
4276         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4277         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4278     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4279         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4280     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4281         qemu_iothread_add($vmid, $deviceid, $device);
4282
4283         qemu_driveadd($storecfg, $vmid, $device);
4284         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4285
4286         qemu_deviceadd($vmid, $devicefull);
4287         eval { qemu_deviceaddverify($vmid, $deviceid); };
4288         if (my $err = $@) {
4289             eval { qemu_drivedel($vmid, $deviceid); };
4290             warn $@ if $@;
4291             die $err;
4292         }
4293     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4294         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4295         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4296         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4297
4298         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4299
4300         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4301             qemu_iothread_add($vmid, $deviceid, $device);
4302             $devicefull .= ",iothread=iothread-$deviceid";
4303         }
4304
4305         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4306             $devicefull .= ",num_queues=$device->{queues}";
4307         }
4308
4309         qemu_deviceadd($vmid, $devicefull);
4310         qemu_deviceaddverify($vmid, $deviceid);
4311     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4312         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4313         qemu_driveadd($storecfg, $vmid, $device);
4314
4315         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4316         eval { qemu_deviceadd($vmid, $devicefull); };
4317         if (my $err = $@) {
4318             eval { qemu_drivedel($vmid, $deviceid); };
4319             warn $@ if $@;
4320             die $err;
4321         }
4322     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4323         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4324
4325         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4326         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4327         my $use_old_bios_files = undef;
4328         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4329
4330         my $netdevicefull = print_netdevice_full(
4331             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4332         qemu_deviceadd($vmid, $netdevicefull);
4333         eval {
4334             qemu_deviceaddverify($vmid, $deviceid);
4335             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4336         };
4337         if (my $err = $@) {
4338             eval { qemu_netdevdel($vmid, $deviceid); };
4339             warn $@ if $@;
4340             die $err;
4341         }
4342     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4343         my $bridgeid = $2;
4344         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4345         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4346
4347         qemu_deviceadd($vmid, $devicefull);
4348         qemu_deviceaddverify($vmid, $deviceid);
4349     } else {
4350         die "can't hotplug device '$deviceid'\n";
4351     }
4352
4353     return 1;
4354 }
4355
4356 # fixme: this should raise exceptions on error!
4357 sub vm_deviceunplug {
4358     my ($vmid, $conf, $deviceid) = @_;
4359
4360     my $devices_list = vm_devices_list($vmid);
4361     return 1 if !defined($devices_list->{$deviceid});
4362
4363     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4364     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4365
4366     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4367         qemu_devicedel($vmid, $deviceid);
4368     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4369         qemu_devicedel($vmid, $deviceid);
4370         qemu_devicedelverify($vmid, $deviceid);
4371     } elsif ($deviceid =~ m/^usb\d+$/) {
4372         qemu_devicedel($vmid, $deviceid);
4373         qemu_devicedelverify($vmid, $deviceid);
4374     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4375         my $device = parse_drive($deviceid, $conf->{$deviceid});
4376
4377         qemu_devicedel($vmid, $deviceid);
4378         qemu_devicedelverify($vmid, $deviceid);
4379         qemu_drivedel($vmid, $deviceid);
4380         qemu_iothread_del($vmid, $deviceid, $device);
4381     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4382         qemu_devicedel($vmid, $deviceid);
4383         qemu_devicedelverify($vmid, $deviceid);
4384     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4385         my $device = parse_drive($deviceid, $conf->{$deviceid});
4386
4387         qemu_devicedel($vmid, $deviceid);
4388         qemu_devicedelverify($vmid, $deviceid);
4389         qemu_drivedel($vmid, $deviceid);
4390         qemu_deletescsihw($conf, $vmid, $deviceid);
4391
4392         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4393             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4394     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4395         qemu_devicedel($vmid, $deviceid);
4396         qemu_devicedelverify($vmid, $deviceid);
4397         qemu_netdevdel($vmid, $deviceid);
4398     } else {
4399         die "can't unplug device '$deviceid'\n";
4400     }
4401
4402     return 1;
4403 }
4404
4405 sub qemu_spice_usbredir_chardev_add {
4406     my ($vmid, $id) = @_;
4407
4408     mon_cmd($vmid, "chardev-add" , (
4409         id => $id,
4410         backend => {
4411             type => 'spicevmc',
4412             data => {
4413                 type => "usbredir",
4414             },
4415         },
4416     ));
4417 }
4418
4419 sub qemu_iothread_add {
4420     my ($vmid, $deviceid, $device) = @_;
4421
4422     if ($device->{iothread}) {
4423         my $iothreads = vm_iothreads_list($vmid);
4424         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4425     }
4426 }
4427
4428 sub qemu_iothread_del {
4429     my ($vmid, $deviceid, $device) = @_;
4430
4431     if ($device->{iothread}) {
4432         my $iothreads = vm_iothreads_list($vmid);
4433         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4434     }
4435 }
4436
4437 sub qemu_driveadd {
4438     my ($storecfg, $vmid, $device) = @_;
4439
4440     my $kvmver = get_running_qemu_version($vmid);
4441     my $io_uring = min_version($kvmver, 6, 0);
4442     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4443     $drive =~ s/\\/\\\\/g;
4444     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4445
4446     # If the command succeeds qemu prints: "OK"
4447     return 1 if $ret =~ m/OK/s;
4448
4449     die "adding drive failed: $ret\n";
4450 }
4451
4452 sub qemu_drivedel {
4453     my ($vmid, $deviceid) = @_;
4454
4455     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4456     $ret =~ s/^\s+//;
4457
4458     return 1 if $ret eq "";
4459
4460     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4461     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4462
4463     die "deleting drive $deviceid failed : $ret\n";
4464 }
4465
4466 sub qemu_deviceaddverify {
4467     my ($vmid, $deviceid) = @_;
4468
4469     for (my $i = 0; $i <= 5; $i++) {
4470          my $devices_list = vm_devices_list($vmid);
4471          return 1 if defined($devices_list->{$deviceid});
4472          sleep 1;
4473     }
4474
4475     die "error on hotplug device '$deviceid'\n";
4476 }
4477
4478
4479 sub qemu_devicedelverify {
4480     my ($vmid, $deviceid) = @_;
4481
4482     # need to verify that the device is correctly removed as device_del
4483     # is async and empty return is not reliable
4484
4485     for (my $i = 0; $i <= 5; $i++) {
4486          my $devices_list = vm_devices_list($vmid);
4487          return 1 if !defined($devices_list->{$deviceid});
4488          sleep 1;
4489     }
4490
4491     die "error on hot-unplugging device '$deviceid'\n";
4492 }
4493
4494 sub qemu_findorcreatescsihw {
4495     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4496
4497     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4498
4499     my $scsihwid="$controller_prefix$controller";
4500     my $devices_list = vm_devices_list($vmid);
4501
4502     if (!defined($devices_list->{$scsihwid})) {
4503         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4504     }
4505
4506     return 1;
4507 }
4508
4509 sub qemu_deletescsihw {
4510     my ($conf, $vmid, $opt) = @_;
4511
4512     my $device = parse_drive($opt, $conf->{$opt});
4513
4514     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4515         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4516         return 1;
4517     }
4518
4519     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4520
4521     my $devices_list = vm_devices_list($vmid);
4522     foreach my $opt (keys %{$devices_list}) {
4523         if (is_valid_drivename($opt)) {
4524             my $drive = parse_drive($opt, $conf->{$opt});
4525             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4526                 return 1;
4527             }
4528         }
4529     }
4530
4531     my $scsihwid="scsihw$controller";
4532
4533     vm_deviceunplug($vmid, $conf, $scsihwid);
4534
4535     return 1;
4536 }
4537
4538 sub qemu_add_pci_bridge {
4539     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4540
4541     my $bridges = {};
4542
4543     my $bridgeid;
4544
4545     print_pci_addr($device, $bridges, $arch, $machine_type);
4546
4547     while (my ($k, $v) = each %$bridges) {
4548         $bridgeid = $k;
4549     }
4550     return 1 if !defined($bridgeid) || $bridgeid < 1;
4551
4552     my $bridge = "pci.$bridgeid";
4553     my $devices_list = vm_devices_list($vmid);
4554
4555     if (!defined($devices_list->{$bridge})) {
4556         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4557     }
4558
4559     return 1;
4560 }
4561
4562 sub qemu_set_link_status {
4563     my ($vmid, $device, $up) = @_;
4564
4565     mon_cmd($vmid, "set_link", name => $device,
4566                up => $up ? JSON::true : JSON::false);
4567 }
4568
4569 sub qemu_netdevadd {
4570     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4571
4572     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4573     my %options =  split(/[=,]/, $netdev);
4574
4575     if (defined(my $vhost = $options{vhost})) {
4576         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4577     }
4578
4579     if (defined(my $queues = $options{queues})) {
4580         $options{queues} = $queues + 0;
4581     }
4582
4583     mon_cmd($vmid, "netdev_add",  %options);
4584     return 1;
4585 }
4586
4587 sub qemu_netdevdel {
4588     my ($vmid, $deviceid) = @_;
4589
4590     mon_cmd($vmid, "netdev_del", id => $deviceid);
4591 }
4592
4593 sub qemu_usb_hotplug {
4594     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4595
4596     return if !$device;
4597
4598     # remove the old one first
4599     vm_deviceunplug($vmid, $conf, $deviceid);
4600
4601     # check if xhci controller is necessary and available
4602     my $devicelist = vm_devices_list($vmid);
4603
4604     if (!$devicelist->{xhci}) {
4605         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4606         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4607     }
4608
4609     # add the new one
4610     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4611 }
4612
4613 sub qemu_cpu_hotplug {
4614     my ($vmid, $conf, $vcpus) = @_;
4615
4616     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4617
4618     my $sockets = 1;
4619     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4620     $sockets = $conf->{sockets} if  $conf->{sockets};
4621     my $cores = $conf->{cores} || 1;
4622     my $maxcpus = $sockets * $cores;
4623
4624     $vcpus = $maxcpus if !$vcpus;
4625
4626     die "you can't add more vcpus than maxcpus\n"
4627         if $vcpus > $maxcpus;
4628
4629     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4630
4631     if ($vcpus < $currentvcpus) {
4632
4633         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4634
4635             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4636                 qemu_devicedel($vmid, "cpu$i");
4637                 my $retry = 0;
4638                 my $currentrunningvcpus = undef;
4639                 while (1) {
4640                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4641                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4642                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4643                     $retry++;
4644                     sleep 1;
4645                 }
4646                 #update conf after each succesfull cpu unplug
4647                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4648                 PVE::QemuConfig->write_config($vmid, $conf);
4649             }
4650         } else {
4651             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4652         }
4653
4654         return;
4655     }
4656
4657     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4658     die "vcpus in running vm does not match its configuration\n"
4659         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4660
4661     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4662
4663         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4664             my $cpustr = print_cpu_device($conf, $i);
4665             qemu_deviceadd($vmid, $cpustr);
4666
4667             my $retry = 0;
4668             my $currentrunningvcpus = undef;
4669             while (1) {
4670                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4671                 last if scalar(@{$currentrunningvcpus}) == $i;
4672                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4673                 sleep 1;
4674                 $retry++;
4675             }
4676             #update conf after each succesfull cpu hotplug
4677             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4678             PVE::QemuConfig->write_config($vmid, $conf);
4679         }
4680     } else {
4681
4682         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4683             mon_cmd($vmid, "cpu-add", id => int($i));
4684         }
4685     }
4686 }
4687
4688 sub qemu_block_set_io_throttle {
4689     my ($vmid, $deviceid,
4690         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4691         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4692         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4693         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4694
4695     return if !check_running($vmid) ;
4696
4697     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4698         bps => int($bps),
4699         bps_rd => int($bps_rd),
4700         bps_wr => int($bps_wr),
4701         iops => int($iops),
4702         iops_rd => int($iops_rd),
4703         iops_wr => int($iops_wr),
4704         bps_max => int($bps_max),
4705         bps_rd_max => int($bps_rd_max),
4706         bps_wr_max => int($bps_wr_max),
4707         iops_max => int($iops_max),
4708         iops_rd_max => int($iops_rd_max),
4709         iops_wr_max => int($iops_wr_max),
4710         bps_max_length => int($bps_max_length),
4711         bps_rd_max_length => int($bps_rd_max_length),
4712         bps_wr_max_length => int($bps_wr_max_length),
4713         iops_max_length => int($iops_max_length),
4714         iops_rd_max_length => int($iops_rd_max_length),
4715         iops_wr_max_length => int($iops_wr_max_length),
4716     );
4717
4718 }
4719
4720 sub qemu_block_resize {
4721     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4722
4723     my $running = check_running($vmid);
4724
4725     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4726
4727     return if !$running;
4728
4729     my $padding = (1024 - $size % 1024) % 1024;
4730     $size = $size + $padding;
4731
4732     mon_cmd(
4733         $vmid,
4734         "block_resize",
4735         device => $deviceid,
4736         size => int($size),
4737         timeout => 60,
4738     );
4739 }
4740
4741 sub qemu_volume_snapshot {
4742     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4743
4744     my $running = check_running($vmid);
4745
4746     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4747         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4748     } else {
4749         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4750     }
4751 }
4752
4753 sub qemu_volume_snapshot_delete {
4754     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4755
4756     my $running = check_running($vmid);
4757
4758     if($running) {
4759
4760         $running = undef;
4761         my $conf = PVE::QemuConfig->load_config($vmid);
4762         PVE::QemuConfig->foreach_volume($conf, sub {
4763             my ($ds, $drive) = @_;
4764             $running = 1 if $drive->{file} eq $volid;
4765         });
4766     }
4767
4768     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4769         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4770     } else {
4771         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4772     }
4773 }
4774
4775 sub set_migration_caps {
4776     my ($vmid, $savevm) = @_;
4777
4778     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4779
4780     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4781     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4782
4783     my $cap_ref = [];
4784
4785     my $enabled_cap = {
4786         "auto-converge" => 1,
4787         "xbzrle" => 1,
4788         "x-rdma-pin-all" => 0,
4789         "zero-blocks" => 0,
4790         "compress" => 0,
4791         "dirty-bitmaps" => $dirty_bitmaps,
4792     };
4793
4794     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4795
4796     for my $supported_capability (@$supported_capabilities) {
4797         push @$cap_ref, {
4798             capability => $supported_capability->{capability},
4799             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4800         };
4801     }
4802
4803     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4804 }
4805
4806 sub foreach_volid {
4807     my ($conf, $func, @param) = @_;
4808
4809     my $volhash = {};
4810
4811     my $test_volid = sub {
4812         my ($key, $drive, $snapname, $pending) = @_;
4813
4814         my $volid = $drive->{file};
4815         return if !$volid;
4816
4817         $volhash->{$volid}->{cdrom} //= 1;
4818         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4819
4820         my $replicate = $drive->{replicate} // 1;
4821         $volhash->{$volid}->{replicate} //= 0;
4822         $volhash->{$volid}->{replicate} = 1 if $replicate;
4823
4824         $volhash->{$volid}->{shared} //= 0;
4825         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4826
4827         $volhash->{$volid}->{is_unused} //= 0;
4828         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4829
4830         $volhash->{$volid}->{is_attached} //= 0;
4831         $volhash->{$volid}->{is_attached} = 1
4832             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4833
4834         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4835             if defined($snapname);
4836
4837         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4838
4839         my $size = $drive->{size};
4840         $volhash->{$volid}->{size} //= $size if $size;
4841
4842         $volhash->{$volid}->{is_vmstate} //= 0;
4843         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4844
4845         $volhash->{$volid}->{is_tpmstate} //= 0;
4846         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4847
4848         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4849     };
4850
4851     my $include_opts = {
4852         extra_keys => ['vmstate'],
4853         include_unused => 1,
4854     };
4855
4856     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4857
4858     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4859         if defined($conf->{pending}) && $conf->{pending}->%*;
4860
4861     foreach my $snapname (keys %{$conf->{snapshots}}) {
4862         my $snap = $conf->{snapshots}->{$snapname};
4863         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4864     }
4865
4866     foreach my $volid (keys %$volhash) {
4867         &$func($volid, $volhash->{$volid}, @param);
4868     }
4869 }
4870
4871 my $fast_plug_option = {
4872     'description' => 1,
4873     'hookscript' => 1,
4874     'lock' => 1,
4875     'migrate_downtime' => 1,
4876     'migrate_speed' => 1,
4877     'name' => 1,
4878     'onboot' => 1,
4879     'protection' => 1,
4880     'shares' => 1,
4881     'startup' => 1,
4882     'tags' => 1,
4883     'vmstatestorage' => 1,
4884 };
4885
4886 for my $opt (keys %$confdesc_cloudinit) {
4887     $fast_plug_option->{$opt} = 1;
4888 };
4889
4890 # hotplug changes in [PENDING]
4891 # $selection hash can be used to only apply specified options, for
4892 # example: { cores => 1 } (only apply changed 'cores')
4893 # $errors ref is used to return error messages
4894 sub vmconfig_hotplug_pending {
4895     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4896
4897     my $defaults = load_defaults();
4898     my $arch = get_vm_arch($conf);
4899     my $machine_type = get_vm_machine($conf, undef, $arch);
4900
4901     # commit values which do not have any impact on running VM first
4902     # Note: those option cannot raise errors, we we do not care about
4903     # $selection and always apply them.
4904
4905     my $add_error = sub {
4906         my ($opt, $msg) = @_;
4907         $errors->{$opt} = "hotplug problem - $msg";
4908     };
4909
4910     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4911
4912     my $cloudinit_record_changed = sub {
4913         my ($conf, $opt, $old, $new) = @_;
4914         return if !$cloudinit_pending_properties->{$opt};
4915
4916         my $ci = ($conf->{cloudinit} //= {});
4917
4918         my $recorded = $ci->{$opt};
4919         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4920
4921         if (defined($new)) {
4922             if (defined($old)) {
4923                 # an existing value is being modified
4924                 if (defined($recorded)) {
4925                     # the value was already not in sync
4926                     if ($new eq $recorded) {
4927                         # a value is being reverted to the cloud-init state:
4928                         delete $ci->{$opt};
4929                         delete $added{$opt};
4930                     } else {
4931                         # the value was changed multiple times, do nothing
4932                     }
4933                 } elsif ($added{$opt}) {
4934                     # the value had been marked as added and is being changed, do nothing
4935                 } else {
4936                     # the value is new, record it:
4937                     $ci->{$opt} = $old;
4938                 }
4939             } else {
4940                 # a new value is being added
4941                 if (defined($recorded)) {
4942                     # it was already not in sync
4943                     if ($new eq $recorded) {
4944                         # a value is being reverted to the cloud-init state:
4945                         delete $ci->{$opt};
4946                         delete $added{$opt};
4947                     } else {
4948                         # the value had temporarily been removed, do nothing
4949                     }
4950                 } elsif ($added{$opt}) {
4951                     # the value had been marked as added already, do nothing
4952                 } else {
4953                     # the value is new, add it
4954                     $added{$opt} = 1;
4955                 }
4956             }
4957         } elsif (!defined($old)) {
4958             # a non-existent value is being removed? ignore...
4959         } else {
4960             # a value is being deleted
4961             if (defined($recorded)) {
4962                 # a value was already recorded, just keep it
4963             } elsif ($added{$opt}) {
4964                 # the value was marked as added, remove it
4965                 delete $added{$opt};
4966             } else {
4967                 # a previously unrecorded value is being removed, record the old value:
4968                 $ci->{$opt} = $old;
4969             }
4970         }
4971
4972         my $added = join(',', sort keys %added);
4973         $ci->{added} = $added if length($added);
4974     };
4975
4976     my $changes = 0;
4977     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4978         if ($fast_plug_option->{$opt}) {
4979             my $new = delete $conf->{pending}->{$opt};
4980             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4981             $conf->{$opt} = $new;
4982             $changes = 1;
4983         }
4984     }
4985
4986     if ($changes) {
4987         PVE::QemuConfig->write_config($vmid, $conf);
4988     }
4989
4990     my $ostype = $conf->{ostype};
4991     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4992     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4993     my $usb_hotplug = $hotplug_features->{usb}
4994         && min_version($version, 7, 1)
4995         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4996
4997     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4998     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4999
5000     foreach my $opt (sort keys %$pending_delete_hash) {
5001         next if $selection && !$selection->{$opt};
5002         my $force = $pending_delete_hash->{$opt}->{force};
5003         eval {
5004             if ($opt eq 'hotplug') {
5005                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
5006             } elsif ($opt eq 'tablet') {
5007                 die "skip\n" if !$hotplug_features->{usb};
5008                 if ($defaults->{tablet}) {
5009                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5010                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5011                         if $arch eq 'aarch64';
5012                 } else {
5013                     vm_deviceunplug($vmid, $conf, 'tablet');
5014                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5015                 }
5016             } elsif ($opt =~ m/^usb(\d+)$/) {
5017                 my $index = $1;
5018                 die "skip\n" if !$usb_hotplug;
5019                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5020                 vm_deviceunplug($vmid, $conf, $opt);
5021             } elsif ($opt eq 'vcpus') {
5022                 die "skip\n" if !$hotplug_features->{cpu};
5023                 qemu_cpu_hotplug($vmid, $conf, undef);
5024             } elsif ($opt eq 'balloon') {
5025                 # enable balloon device is not hotpluggable
5026                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5027                 # here we reset the ballooning value to memory
5028                 my $balloon = get_current_memory($conf->{memory});
5029                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5030             } elsif ($fast_plug_option->{$opt}) {
5031                 # do nothing
5032             } elsif ($opt =~ m/^net(\d+)$/) {
5033                 die "skip\n" if !$hotplug_features->{network};
5034                 vm_deviceunplug($vmid, $conf, $opt);
5035                 if($have_sdn) {
5036                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5037                     PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
5038                 }
5039             } elsif (is_valid_drivename($opt)) {
5040                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5041                 vm_deviceunplug($vmid, $conf, $opt);
5042                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5043             } elsif ($opt =~ m/^memory$/) {
5044                 die "skip\n" if !$hotplug_features->{memory};
5045                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
5046             } elsif ($opt eq 'cpuunits') {
5047                 $cgroup->change_cpu_shares(undef);
5048             } elsif ($opt eq 'cpulimit') {
5049                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5050             } else {
5051                 die "skip\n";
5052             }
5053         };
5054         if (my $err = $@) {
5055             &$add_error($opt, $err) if $err ne "skip\n";
5056         } else {
5057             my $old = delete $conf->{$opt};
5058             $cloudinit_record_changed->($conf, $opt, $old, undef);
5059             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5060         }
5061     }
5062
5063     my $cloudinit_opt;
5064     foreach my $opt (keys %{$conf->{pending}}) {
5065         next if $selection && !$selection->{$opt};
5066         my $value = $conf->{pending}->{$opt};
5067         eval {
5068             if ($opt eq 'hotplug') {
5069                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5070                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
5071             } elsif ($opt eq 'tablet') {
5072                 die "skip\n" if !$hotplug_features->{usb};
5073                 if ($value == 1) {
5074                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5075                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5076                         if $arch eq 'aarch64';
5077                 } elsif ($value == 0) {
5078                     vm_deviceunplug($vmid, $conf, 'tablet');
5079                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5080                 }
5081             } elsif ($opt =~ m/^usb(\d+)$/) {
5082                 my $index = $1;
5083                 die "skip\n" if !$usb_hotplug;
5084                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5085                 my $id = $opt;
5086                 if ($d->{host} =~ m/^spice$/i)  {
5087                     $id = "usbredirdev$index";
5088                 }
5089                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5090             } elsif ($opt eq 'vcpus') {
5091                 die "skip\n" if !$hotplug_features->{cpu};
5092                 qemu_cpu_hotplug($vmid, $conf, $value);
5093             } elsif ($opt eq 'balloon') {
5094                 # enable/disable balloning device is not hotpluggable
5095                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5096                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5097                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5098
5099                 # allow manual ballooning if shares is set to zero
5100                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5101                     my $memory = get_current_memory($conf->{memory});
5102                     my $balloon = $conf->{pending}->{balloon} || $memory;
5103                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5104                 }
5105             } elsif ($opt =~ m/^net(\d+)$/) {
5106                 # some changes can be done without hotplug
5107                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5108                                     $vmid, $opt, $value, $arch, $machine_type);
5109             } elsif (is_valid_drivename($opt)) {
5110                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5111                 # some changes can be done without hotplug
5112                 my $drive = parse_drive($opt, $value);
5113                 if (drive_is_cloudinit($drive)) {
5114                     $cloudinit_opt = [$opt, $drive];
5115                     # apply all the other changes first, then generate the cloudinit disk
5116                     die "skip\n";
5117                 }
5118                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5119                                      $vmid, $opt, $value, $arch, $machine_type);
5120             } elsif ($opt =~ m/^memory$/) { #dimms
5121                 die "skip\n" if !$hotplug_features->{memory};
5122                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5123             } elsif ($opt eq 'cpuunits') {
5124                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5125                 $cgroup->change_cpu_shares($new_cpuunits);
5126             } elsif ($opt eq 'cpulimit') {
5127                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5128                 $cgroup->change_cpu_quota($cpulimit, 100000);
5129             } elsif ($opt eq 'agent') {
5130                 vmconfig_update_agent($conf, $opt, $value);
5131             } else {
5132                 die "skip\n";  # skip non-hot-pluggable options
5133             }
5134         };
5135         if (my $err = $@) {
5136             &$add_error($opt, $err) if $err ne "skip\n";
5137         } else {
5138             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5139             $conf->{$opt} = $value;
5140             delete $conf->{pending}->{$opt};
5141         }
5142     }
5143
5144     if (defined($cloudinit_opt)) {
5145         my ($opt, $drive) = @$cloudinit_opt;
5146         my $value = $conf->{pending}->{$opt};
5147         eval {
5148             my $temp = {%$conf, $opt => $value};
5149             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5150             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5151                                  $vmid, $opt, $value, $arch, $machine_type);
5152         };
5153         if (my $err = $@) {
5154             &$add_error($opt, $err) if $err ne "skip\n";
5155         } else {
5156             $conf->{$opt} = $value;
5157             delete $conf->{pending}->{$opt};
5158         }
5159     }
5160
5161     # unplug xhci controller if no usb device is left
5162     if ($usb_hotplug) {
5163         my $has_usb = 0;
5164         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5165             next if !defined($conf->{"usb$i"});
5166             $has_usb = 1;
5167             last;
5168         }
5169         if (!$has_usb) {
5170             vm_deviceunplug($vmid, $conf, 'xhci');
5171         }
5172     }
5173
5174     PVE::QemuConfig->write_config($vmid, $conf);
5175
5176     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5177         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5178     }
5179 }
5180
5181 sub try_deallocate_drive {
5182     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5183
5184     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5185         my $volid = $drive->{file};
5186         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5187             my $sid = PVE::Storage::parse_volume_id($volid);
5188             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5189
5190             # check if the disk is really unused
5191             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5192                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5193             PVE::Storage::vdisk_free($storecfg, $volid);
5194             return 1;
5195         } else {
5196             # If vm is not owner of this disk remove from config
5197             return 1;
5198         }
5199     }
5200
5201     return;
5202 }
5203
5204 sub vmconfig_delete_or_detach_drive {
5205     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5206
5207     my $drive = parse_drive($opt, $conf->{$opt});
5208
5209     my $rpcenv = PVE::RPCEnvironment::get();
5210     my $authuser = $rpcenv->get_user();
5211
5212     if ($force) {
5213         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5214         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5215     } else {
5216         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5217     }
5218 }
5219
5220
5221
5222 sub vmconfig_apply_pending {
5223     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5224
5225     return if !scalar(keys %{$conf->{pending}});
5226
5227     my $add_apply_error = sub {
5228         my ($opt, $msg) = @_;
5229         my $err_msg = "unable to apply pending change $opt : $msg";
5230         $errors->{$opt} = $err_msg;
5231         warn $err_msg;
5232     };
5233
5234     # cold plug
5235
5236     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5237     foreach my $opt (sort keys %$pending_delete_hash) {
5238         my $force = $pending_delete_hash->{$opt}->{force};
5239         eval {
5240             if ($opt =~ m/^unused/) {
5241                 die "internal error";
5242             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5243                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5244             } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5245                 if($have_sdn) {
5246                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5247                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5248                     warn if $@;
5249                 }
5250             }
5251         };
5252         if (my $err = $@) {
5253             $add_apply_error->($opt, $err);
5254         } else {
5255             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5256             delete $conf->{$opt};
5257         }
5258     }
5259
5260     PVE::QemuConfig->cleanup_pending($conf);
5261
5262     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5263
5264     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5265         next if $opt eq 'delete'; # just to be sure
5266         eval {
5267             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5268                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5269             } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5270                 if($have_sdn) {
5271                     my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5272                     if ($conf->{$opt}){
5273                         my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5274
5275                         if ($old_net->{bridge} ne $new_net->{bridge} ||
5276                             $old_net->{macaddr} ne $new_net->{macaddr}) {
5277                             PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5278                         }
5279                    }
5280                    #fixme: reuse ip if mac change && same bridge
5281                    PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5282                 }
5283             }
5284         };
5285         if (my $err = $@) {
5286             $add_apply_error->($opt, $err);
5287         } else {
5288
5289             if (is_valid_drivename($opt)) {
5290                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5291                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5292             }
5293
5294             $conf->{$opt} = delete $conf->{pending}->{$opt};
5295         }
5296     }
5297
5298     # write all changes at once to avoid unnecessary i/o
5299     PVE::QemuConfig->write_config($vmid, $conf);
5300     if ($generate_cloudinit) {
5301         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5302             # After successful generation and if there were changes to be applied, update the
5303             # config to drop the {cloudinit} entry.
5304             PVE::QemuConfig->write_config($vmid, $conf);
5305         }
5306     }
5307 }
5308
5309 sub vmconfig_update_net {
5310     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5311
5312     my $newnet = parse_net($value);
5313
5314     if ($conf->{$opt}) {
5315         my $oldnet = parse_net($conf->{$opt});
5316
5317         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5318             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5319             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5320             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5321             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5322
5323             # for non online change, we try to hot-unplug
5324             die "skip\n" if !$hotplug;
5325             vm_deviceunplug($vmid, $conf, $opt);
5326
5327             if($have_sdn) {
5328                 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5329             }
5330
5331         } else {
5332
5333             die "internal error" if $opt !~ m/net(\d+)/;
5334             my $iface = "tap${vmid}i$1";
5335
5336             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5337                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5338                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5339                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5340                 PVE::Network::tap_unplug($iface);
5341
5342                 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5343                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5344                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5345                     qemu_set_link_status($vmid, $opt, 0);
5346                 }
5347
5348                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5349                     if ($have_sdn) {
5350                         PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5351                         PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5352                     }
5353                 }
5354
5355                 if ($have_sdn) {
5356                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5357                 } else {
5358                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5359                 }
5360
5361                 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5362                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5363                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5364                     qemu_set_link_status($vmid, $opt, 1);
5365                 }
5366
5367             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5368                 # Rate can be applied on its own but any change above needs to
5369                 # include the rate in tap_plug since OVS resets everything.
5370                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5371             }
5372
5373             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5374                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5375             }
5376
5377             return 1;
5378         }
5379     }
5380
5381     if ($hotplug) {
5382         if ($have_sdn) {
5383             PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5384             PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5385         }
5386         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5387     } else {
5388         die "skip\n";
5389     }
5390 }
5391
5392 sub vmconfig_update_agent {
5393     my ($conf, $opt, $value) = @_;
5394
5395     die "skip\n" if !$conf->{$opt};
5396
5397     my $hotplug_options = { fstrim_cloned_disks => 1 };
5398
5399     my $old_agent = parse_guest_agent($conf);
5400     my $agent = parse_guest_agent({$opt => $value});
5401
5402     for my $option (keys %$agent) { # added/changed options
5403         next if defined($hotplug_options->{$option});
5404         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5405     }
5406
5407     for my $option (keys %$old_agent) { # removed options
5408         next if defined($hotplug_options->{$option});
5409         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5410     }
5411
5412     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5413 }
5414
5415 sub vmconfig_update_disk {
5416     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5417
5418     my $drive = parse_drive($opt, $value);
5419
5420     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5421         my $media = $drive->{media} || 'disk';
5422         my $oldmedia = $old_drive->{media} || 'disk';
5423         die "unable to change media type\n" if $media ne $oldmedia;
5424
5425         if (!drive_is_cdrom($old_drive)) {
5426
5427             if ($drive->{file} ne $old_drive->{file}) {
5428
5429                 die "skip\n" if !$hotplug;
5430
5431                 # unplug and register as unused
5432                 vm_deviceunplug($vmid, $conf, $opt);
5433                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5434
5435             } else {
5436                 # update existing disk
5437
5438                 # skip non hotpluggable value
5439                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5440                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5441                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5442                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5443                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5444                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5445                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5446                     die "skip\n";
5447                 }
5448
5449                 # apply throttle
5450                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5451                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5452                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5453                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5454                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5455                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5456                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5457                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5458                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5459                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5460                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5461                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5462                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5463                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5464                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5465                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5466                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5467                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5468
5469                     qemu_block_set_io_throttle(
5470                         $vmid,"drive-$opt",
5471                         ($drive->{mbps} || 0)*1024*1024,
5472                         ($drive->{mbps_rd} || 0)*1024*1024,
5473                         ($drive->{mbps_wr} || 0)*1024*1024,
5474                          $drive->{iops} || 0,
5475                          $drive->{iops_rd} || 0,
5476                          $drive->{iops_wr} || 0,
5477                         ($drive->{mbps_max} || 0)*1024*1024,
5478                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5479                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5480                          $drive->{iops_max} || 0,
5481                          $drive->{iops_rd_max} || 0,
5482                          $drive->{iops_wr_max} || 0,
5483                          $drive->{bps_max_length} || 1,
5484                          $drive->{bps_rd_max_length} || 1,
5485                          $drive->{bps_wr_max_length} || 1,
5486                          $drive->{iops_max_length} || 1,
5487                          $drive->{iops_rd_max_length} || 1,
5488                          $drive->{iops_wr_max_length} || 1,
5489                     );
5490
5491                 }
5492
5493                 return 1;
5494             }
5495
5496         } else { # cdrom
5497
5498             if ($drive->{file} eq 'none') {
5499                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5500                 if (drive_is_cloudinit($old_drive)) {
5501                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5502                 }
5503             } else {
5504                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5505
5506                 # force eject if locked
5507                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5508
5509                 if ($path) {
5510                     mon_cmd($vmid, "blockdev-change-medium",
5511                         id => "$opt", filename => "$path");
5512                 }
5513             }
5514
5515             return 1;
5516         }
5517     }
5518
5519     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5520     # hotplug new disks
5521     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5522     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5523 }
5524
5525 sub vmconfig_update_cloudinit_drive {
5526     my ($storecfg, $conf, $vmid) = @_;
5527
5528     my $cloudinit_ds = undef;
5529     my $cloudinit_drive = undef;
5530
5531     PVE::QemuConfig->foreach_volume($conf, sub {
5532         my ($ds, $drive) = @_;
5533         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5534             $cloudinit_ds = $ds;
5535             $cloudinit_drive = $drive;
5536         }
5537     });
5538
5539     return if !$cloudinit_drive;
5540
5541     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5542         PVE::QemuConfig->write_config($vmid, $conf);
5543     }
5544
5545     my $running = PVE::QemuServer::check_running($vmid);
5546
5547     if ($running) {
5548         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5549         if ($path) {
5550             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5551             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5552         }
5553     }
5554 }
5555
5556 # called in locked context by incoming migration
5557 sub vm_migrate_get_nbd_disks {
5558     my ($storecfg, $conf, $replicated_volumes) = @_;
5559
5560     my $local_volumes = {};
5561     PVE::QemuConfig->foreach_volume($conf, sub {
5562         my ($ds, $drive) = @_;
5563
5564         return if drive_is_cdrom($drive);
5565         return if $ds eq 'tpmstate0';
5566
5567         my $volid = $drive->{file};
5568
5569         return if !$volid;
5570
5571         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5572
5573         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5574         return if $scfg->{shared};
5575
5576         my $format = qemu_img_format($scfg, $volname);
5577
5578         # replicated disks re-use existing state via bitmap
5579         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5580         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5581     });
5582     return $local_volumes;
5583 }
5584
5585 # called in locked context by incoming migration
5586 sub vm_migrate_alloc_nbd_disks {
5587     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5588
5589     my $nbd = {};
5590     foreach my $opt (sort keys %$source_volumes) {
5591         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5592
5593         if ($use_existing) {
5594             $nbd->{$opt}->{drivestr} = print_drive($drive);
5595             $nbd->{$opt}->{volid} = $volid;
5596             $nbd->{$opt}->{replicated} = 1;
5597             next;
5598         }
5599
5600         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5601
5602         # order of precedence, filtered by whether storage supports it:
5603         # 1. explicit requested format
5604         # 2. default format of storage
5605         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5606         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5607
5608         my $size = $drive->{size} / 1024;
5609         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5610         my $newdrive = $drive;
5611         $newdrive->{format} = $format;
5612         $newdrive->{file} = $newvolid;
5613         my $drivestr = print_drive($newdrive);
5614         $nbd->{$opt}->{drivestr} = $drivestr;
5615         $nbd->{$opt}->{volid} = $newvolid;
5616     }
5617
5618     return $nbd;
5619 }
5620
5621 # see vm_start_nolock for parameters, additionally:
5622 # migrate_opts:
5623 #   storagemap = parsed storage map for allocating NBD disks
5624 sub vm_start {
5625     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5626
5627     return PVE::QemuConfig->lock_config($vmid, sub {
5628         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5629
5630         die "you can't start a vm if it's a template\n"
5631             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5632
5633         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5634         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5635
5636         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5637
5638         if ($has_backup_lock && $running) {
5639             # a backup is currently running, attempt to start the guest in the
5640             # existing QEMU instance
5641             return vm_resume($vmid);
5642         }
5643
5644         PVE::QemuConfig->check_lock($conf)
5645             if !($params->{skiplock} || $has_suspended_lock);
5646
5647         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5648
5649         die "VM $vmid already running\n" if $running;
5650
5651         if (my $storagemap = $migrate_opts->{storagemap}) {
5652             my $replicated = $migrate_opts->{replicated_volumes};
5653             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5654             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5655
5656             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5657                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5658             }
5659         }
5660
5661         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5662     });
5663 }
5664
5665
5666 # params:
5667 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5668 #   skiplock => 0/1, skip checking for config lock
5669 #   skiptemplate => 0/1, skip checking whether VM is template
5670 #   forcemachine => to force QEMU machine (rollback/migration)
5671 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5672 #   timeout => in seconds
5673 #   paused => start VM in paused state (backup)
5674 #   resume => resume from hibernation
5675 #   pbs-backing => {
5676 #      sata0 => {
5677 #         repository
5678 #         snapshot
5679 #         keyfile
5680 #         archive
5681 #      },
5682 #      virtio2 => ...
5683 #   }
5684 # migrate_opts:
5685 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5686 #   migratedfrom => source node
5687 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5688 #   network => CIDR of migration network
5689 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5690 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5691 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5692 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5693 #       contained in config
5694 sub vm_start_nolock {
5695     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5696
5697     my $statefile = $params->{statefile};
5698     my $resume = $params->{resume};
5699
5700     my $migratedfrom = $migrate_opts->{migratedfrom};
5701     my $migration_type = $migrate_opts->{type};
5702
5703     my $res = {};
5704
5705     # clean up leftover reboot request files
5706     eval { clear_reboot_request($vmid); };
5707     warn $@ if $@;
5708
5709     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5710         vmconfig_apply_pending($vmid, $conf, $storecfg);
5711         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5712     }
5713
5714     # don't regenerate the ISO if the VM is started as part of a live migration
5715     # this way we can reuse the old ISO with the correct config
5716     if (!$migratedfrom) {
5717         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5718             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5719             # $conf->{cloudinit}, so we could just not do this?
5720             # But we do it above, so for now let's be consistent.
5721             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5722         }
5723     }
5724
5725     # override offline migrated volumes, conf is out of date still
5726     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5727         for my $key (sort keys $offline_volumes->%*) {
5728             my $parsed = parse_drive($key, $conf->{$key});
5729             $parsed->{file} = $offline_volumes->{$key};
5730             $conf->{$key} = print_drive($parsed);
5731         }
5732     }
5733
5734     my $defaults = load_defaults();
5735
5736     # set environment variable useful inside network script
5737     # for remote migration the config is available on the target node!
5738     if (!$migrate_opts->{remote_node}) {
5739         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5740     }
5741
5742     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5743
5744     my $forcemachine = $params->{forcemachine};
5745     my $forcecpu = $params->{forcecpu};
5746     if ($resume) {
5747         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5748         $forcemachine = $conf->{runningmachine};
5749         $forcecpu = $conf->{runningcpu};
5750         print "Resuming suspended VM\n";
5751     }
5752
5753     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5754         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5755
5756     my $migration_ip;
5757     my $get_migration_ip = sub {
5758         my ($nodename) = @_;
5759
5760         return $migration_ip if defined($migration_ip);
5761
5762         my $cidr = $migrate_opts->{network};
5763
5764         if (!defined($cidr)) {
5765             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5766             $cidr = $dc_conf->{migration}->{network};
5767         }
5768
5769         if (defined($cidr)) {
5770             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5771
5772             die "could not get IP: no address configured on local " .
5773                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5774
5775             die "could not get IP: multiple addresses configured on local " .
5776                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5777
5778             $migration_ip = @$ips[0];
5779         }
5780
5781         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5782             if !defined($migration_ip);
5783
5784         return $migration_ip;
5785     };
5786
5787     if ($statefile) {
5788         if ($statefile eq 'tcp') {
5789             my $migrate = $res->{migrate} = { proto => 'tcp' };
5790             $migrate->{addr} = "localhost";
5791             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5792             my $nodename = nodename();
5793
5794             if (!defined($migration_type)) {
5795                 if (defined($datacenterconf->{migration}->{type})) {
5796                     $migration_type = $datacenterconf->{migration}->{type};
5797                 } else {
5798                     $migration_type = 'secure';
5799                 }
5800             }
5801
5802             if ($migration_type eq 'insecure') {
5803                 $migrate->{addr} = $get_migration_ip->($nodename);
5804                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5805             }
5806
5807             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5808             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5809             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5810             push @$cmd, '-incoming', $migrate->{uri};
5811             push @$cmd, '-S';
5812
5813         } elsif ($statefile eq 'unix') {
5814             # should be default for secure migrations as a ssh TCP forward
5815             # tunnel is not deterministic reliable ready and fails regurarly
5816             # to set up in time, so use UNIX socket forwards
5817             my $migrate = $res->{migrate} = { proto => 'unix' };
5818             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5819             unlink $migrate->{addr};
5820
5821             $migrate->{uri} = "unix:$migrate->{addr}";
5822             push @$cmd, '-incoming', $migrate->{uri};
5823             push @$cmd, '-S';
5824
5825         } elsif (-e $statefile) {
5826             push @$cmd, '-loadstate', $statefile;
5827         } else {
5828             my $statepath = PVE::Storage::path($storecfg, $statefile);
5829             push @$vollist, $statefile;
5830             push @$cmd, '-loadstate', $statepath;
5831         }
5832     } elsif ($params->{paused}) {
5833         push @$cmd, '-S';
5834     }
5835
5836     my $memory = get_current_memory($conf->{memory});
5837     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5838
5839     my $pci_reserve_list = [];
5840     for my $device (values $pci_devices->%*) {
5841         next if $device->{mdev}; # we don't reserve for mdev devices
5842         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5843     }
5844
5845     # reserve all PCI IDs before actually doing anything with them
5846     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5847
5848     eval {
5849         my $uuid;
5850         for my $id (sort keys %$pci_devices) {
5851             my $d = $pci_devices->{$id};
5852             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5853
5854             my $chosen_mdev;
5855             for my $dev ($d->{ids}->@*) {
5856                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5857                 if ($d->{mdev}) {
5858                     warn $@ if $@;
5859                     $chosen_mdev = $info;
5860                     last if $chosen_mdev; # if successful, we're done
5861                 } else {
5862                     die $@ if $@;
5863                 }
5864             }
5865
5866             next if !$d->{mdev};
5867             die "could not create mediated device\n" if !defined($chosen_mdev);
5868
5869             # nvidia grid needs the uuid of the mdev as qemu parameter
5870             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5871                 if (defined($conf->{smbios1})) {
5872                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5873                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5874                 }
5875                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5876             }
5877         }
5878         push @$cmd, '-uuid', $uuid if defined($uuid);
5879     };
5880     if (my $err = $@) {
5881         eval { cleanup_pci_devices($vmid, $conf) };
5882         warn $@ if $@;
5883         die $err;
5884     }
5885
5886     PVE::Storage::activate_volumes($storecfg, $vollist);
5887
5888
5889     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5890     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5891     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5892     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5893     # timeout should be more than enough here...
5894     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5895
5896     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5897
5898     my %run_params = (
5899         timeout => $statefile ? undef : $start_timeout,
5900         umask => 0077,
5901         noerr => 1,
5902     );
5903
5904     # when migrating, prefix QEMU output so other side can pick up any
5905     # errors that might occur and show the user
5906     if ($migratedfrom) {
5907         $run_params{quiet} = 1;
5908         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5909     }
5910
5911     my %systemd_properties = (
5912         Slice => 'qemu.slice',
5913         KillMode => 'process',
5914         SendSIGKILL => 0,
5915         TimeoutStopUSec => ULONG_MAX, # infinity
5916     );
5917
5918     if (PVE::CGroup::cgroup_mode() == 2) {
5919         $systemd_properties{CPUWeight} = $cpuunits;
5920     } else {
5921         $systemd_properties{CPUShares} = $cpuunits;
5922     }
5923
5924     if (my $cpulimit = $conf->{cpulimit}) {
5925         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5926     }
5927     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5928
5929     my $run_qemu = sub {
5930         PVE::Tools::run_fork sub {
5931             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5932
5933             my $tpmpid;
5934             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5935                 # start the TPM emulator so QEMU can connect on start
5936                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5937             }
5938
5939             my $exitcode = run_command($cmd, %run_params);
5940             if ($exitcode) {
5941                 if ($tpmpid) {
5942                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5943                     kill 'TERM', $tpmpid;
5944                 }
5945                 die "QEMU exited with code $exitcode\n";
5946             }
5947         };
5948     };
5949
5950     if ($conf->{hugepages}) {
5951
5952         my $code = sub {
5953             my $hotplug_features =
5954                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5955             my $hugepages_topology =
5956                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5957
5958             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5959
5960             PVE::QemuServer::Memory::hugepages_mount();
5961             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5962
5963             eval { $run_qemu->() };
5964             if (my $err = $@) {
5965                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5966                     if !$conf->{keephugepages};
5967                 die $err;
5968             }
5969
5970             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5971                 if !$conf->{keephugepages};
5972         };
5973         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5974
5975     } else {
5976         eval { $run_qemu->() };
5977     }
5978
5979     if (my $err = $@) {
5980         # deactivate volumes if start fails
5981         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5982         warn $@ if $@;
5983         eval { cleanup_pci_devices($vmid, $conf) };
5984         warn $@ if $@;
5985
5986         die "start failed: $err";
5987     }
5988
5989     # re-reserve all PCI IDs now that we can know the actual VM PID
5990     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5991     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5992     warn $@ if $@;
5993
5994     if (defined($res->{migrate})) {
5995         print "migration listens on $res->{migrate}->{uri}\n";
5996     } elsif ($statefile) {
5997         eval { mon_cmd($vmid, "cont"); };
5998         warn $@ if $@;
5999     }
6000
6001     #start nbd server for storage migration
6002     if (my $nbd = $migrate_opts->{nbd}) {
6003         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
6004
6005         my $migrate_storage_uri;
6006         # nbd_protocol_version > 0 for unix socket support
6007         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
6008             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
6009             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
6010             $migrate_storage_uri = "nbd:unix:$socket_path";
6011             $res->{migrate}->{unix_sockets} = [$socket_path];
6012         } else {
6013             my $nodename = nodename();
6014             my $localip = $get_migration_ip->($nodename);
6015             my $pfamily = PVE::Tools::get_host_address_family($nodename);
6016             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
6017
6018             mon_cmd($vmid, "nbd-server-start", addr => {
6019                 type => 'inet',
6020                 data => {
6021                     host => "${localip}",
6022                     port => "${storage_migrate_port}",
6023                 },
6024             });
6025             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
6026             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
6027         }
6028
6029         my $block_info = mon_cmd($vmid, "query-block");
6030         $block_info = { map { $_->{device} => $_ } $block_info->@* };
6031
6032         foreach my $opt (sort keys %$nbd) {
6033             my $drivestr = $nbd->{$opt}->{drivestr};
6034             my $volid = $nbd->{$opt}->{volid};
6035
6036             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
6037
6038             mon_cmd(
6039                 $vmid,
6040                 "block-export-add",
6041                 id => "drive-$opt",
6042                 'node-name' => $block_node,
6043                 writable => JSON::true,
6044                 type => "nbd",
6045                 name => "drive-$opt", # NBD export name
6046             );
6047
6048             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6049             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6050             print "re-using replicated volume: $opt - $volid\n"
6051                 if $nbd->{$opt}->{replicated};
6052
6053             $res->{drives}->{$opt} = $nbd->{$opt};
6054             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6055         }
6056     }
6057
6058     if ($migratedfrom) {
6059         eval {
6060             set_migration_caps($vmid);
6061         };
6062         warn $@ if $@;
6063
6064         if ($spice_port) {
6065             print "spice listens on port $spice_port\n";
6066             $res->{spice_port} = $spice_port;
6067             if ($migrate_opts->{spice_ticket}) {
6068                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6069                     $migrate_opts->{spice_ticket});
6070                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6071             }
6072         }
6073
6074     } else {
6075         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6076             if !$statefile && $conf->{balloon};
6077
6078         foreach my $opt (keys %$conf) {
6079             next if $opt !~  m/^net\d+$/;
6080             my $nicconf = parse_net($conf->{$opt});
6081             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6082         }
6083         add_nets_bridge_fdb($conf, $vmid);
6084     }
6085
6086    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6087         eval {
6088             mon_cmd(
6089                 $vmid,
6090                 'qom-set',
6091                 path => "machine/peripheral/balloon0",
6092                 property => "guest-stats-polling-interval",
6093                 value => 2
6094             );
6095         };
6096         log_warn("could not set polling interval for ballooning - $@") if $@;
6097     }
6098
6099     if ($resume) {
6100         print "Resumed VM, removing state\n";
6101         if (my $vmstate = $conf->{vmstate}) {
6102             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6103             PVE::Storage::vdisk_free($storecfg, $vmstate);
6104         }
6105         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6106         PVE::QemuConfig->write_config($vmid, $conf);
6107     }
6108
6109     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6110
6111     my ($current_machine, $is_deprecated) =
6112         PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6113     if ($is_deprecated) {
6114         log_warn(
6115             "current machine version '$current_machine' is deprecated - see the documentation and ".
6116             "change to a newer one",
6117         );
6118     }
6119
6120     return $res;
6121 }
6122
6123 sub vm_commandline {
6124     my ($storecfg, $vmid, $snapname) = @_;
6125
6126     my $conf = PVE::QemuConfig->load_config($vmid);
6127
6128     my ($forcemachine, $forcecpu);
6129     if ($snapname) {
6130         my $snapshot = $conf->{snapshots}->{$snapname};
6131         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6132
6133         # check for machine or CPU overrides in snapshot
6134         $forcemachine = $snapshot->{runningmachine};
6135         $forcecpu = $snapshot->{runningcpu};
6136
6137         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6138
6139         $conf = $snapshot;
6140     }
6141
6142     my $defaults = load_defaults();
6143
6144     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6145
6146     return PVE::Tools::cmd2string($cmd);
6147 }
6148
6149 sub vm_reset {
6150     my ($vmid, $skiplock) = @_;
6151
6152     PVE::QemuConfig->lock_config($vmid, sub {
6153
6154         my $conf = PVE::QemuConfig->load_config($vmid);
6155
6156         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6157
6158         mon_cmd($vmid, "system_reset");
6159     });
6160 }
6161
6162 sub get_vm_volumes {
6163     my ($conf) = @_;
6164
6165     my $vollist = [];
6166     foreach_volid($conf, sub {
6167         my ($volid, $attr) = @_;
6168
6169         return if $volid =~ m|^/|;
6170
6171         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6172         return if !$sid;
6173
6174         push @$vollist, $volid;
6175     });
6176
6177     return $vollist;
6178 }
6179
6180 sub cleanup_pci_devices {
6181     my ($vmid, $conf) = @_;
6182
6183     foreach my $key (keys %$conf) {
6184         next if $key !~ m/^hostpci(\d+)$/;
6185         my $hostpciindex = $1;
6186         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6187         my $d = parse_hostpci($conf->{$key});
6188         if ($d->{mdev}) {
6189             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6190             # don't want to break ABI just for this two liner
6191             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6192
6193             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6194             # out when we do it first. so wait for 10 seconds and then try it
6195             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/) {
6196                 sleep 10;
6197             }
6198
6199             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6200         }
6201     }
6202     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6203 }
6204
6205 sub vm_stop_cleanup {
6206     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6207
6208     eval {
6209
6210         if (!$keepActive) {
6211             my $vollist = get_vm_volumes($conf);
6212             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6213
6214             if (my $tpmdrive = $conf->{tpmstate0}) {
6215                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6216                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6217                 if ($storeid) {
6218                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6219                 }
6220             }
6221         }
6222
6223         foreach my $ext (qw(mon qmp pid vnc qga)) {
6224             unlink "/var/run/qemu-server/${vmid}.$ext";
6225         }
6226
6227         if ($conf->{ivshmem}) {
6228             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6229             # just delete it for now, VMs which have this already open do not
6230             # are affected, but new VMs will get a separated one. If this
6231             # becomes an issue we either add some sort of ref-counting or just
6232             # add a "don't delete on stop" flag to the ivshmem format.
6233             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6234         }
6235
6236         cleanup_pci_devices($vmid, $conf);
6237
6238         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6239     };
6240     warn $@ if $@; # avoid errors - just warn
6241 }
6242
6243 # call only in locked context
6244 sub _do_vm_stop {
6245     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6246
6247     my $pid = check_running($vmid, $nocheck);
6248     return if !$pid;
6249
6250     my $conf;
6251     if (!$nocheck) {
6252         $conf = PVE::QemuConfig->load_config($vmid);
6253         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6254         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6255             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6256             $timeout = $opts->{down} if $opts->{down};
6257         }
6258         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6259     }
6260
6261     eval {
6262         if ($shutdown) {
6263             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6264                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6265             } else {
6266                 mon_cmd($vmid, "system_powerdown");
6267             }
6268         } else {
6269             mon_cmd($vmid, "quit");
6270         }
6271     };
6272     my $err = $@;
6273
6274     if (!$err) {
6275         $timeout = 60 if !defined($timeout);
6276
6277         my $count = 0;
6278         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6279             $count++;
6280             sleep 1;
6281         }
6282
6283         if ($count >= $timeout) {
6284             if ($force) {
6285                 warn "VM still running - terminating now with SIGTERM\n";
6286                 kill 15, $pid;
6287             } else {
6288                 die "VM quit/powerdown failed - got timeout\n";
6289             }
6290         } else {
6291             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6292             return;
6293         }
6294     } else {
6295        if (!check_running($vmid, $nocheck)) {
6296             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6297             return;
6298        }
6299        if ($force) {
6300             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6301             kill 15, $pid;
6302         } else {
6303             die "VM quit/powerdown failed\n";
6304         }
6305     }
6306
6307     # wait again
6308     $timeout = 10;
6309
6310     my $count = 0;
6311     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6312         $count++;
6313         sleep 1;
6314     }
6315
6316     if ($count >= $timeout) {
6317         warn "VM still running - terminating now with SIGKILL\n";
6318         kill 9, $pid;
6319         sleep 1;
6320     }
6321
6322     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6323 }
6324
6325 # Note: use $nocheck to skip tests if VM configuration file exists.
6326 # We need that when migration VMs to other nodes (files already moved)
6327 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6328 sub vm_stop {
6329     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6330
6331     $force = 1 if !defined($force) && !$shutdown;
6332
6333     if ($migratedfrom){
6334         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6335         kill 15, $pid if $pid;
6336         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6337         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6338         return;
6339     }
6340
6341     PVE::QemuConfig->lock_config($vmid, sub {
6342         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6343    });
6344 }
6345
6346 sub vm_reboot {
6347     my ($vmid, $timeout) = @_;
6348
6349     PVE::QemuConfig->lock_config($vmid, sub {
6350         eval {
6351
6352             # only reboot if running, as qmeventd starts it again on a stop event
6353             return if !check_running($vmid);
6354
6355             create_reboot_request($vmid);
6356
6357             my $storecfg = PVE::Storage::config();
6358             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6359
6360         };
6361         if (my $err = $@) {
6362             # avoid that the next normal shutdown will be confused for a reboot
6363             clear_reboot_request($vmid);
6364             die $err;
6365         }
6366    });
6367 }
6368
6369 # note: if using the statestorage parameter, the caller has to check privileges
6370 sub vm_suspend {
6371     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6372
6373     my $conf;
6374     my $path;
6375     my $storecfg;
6376     my $vmstate;
6377
6378     PVE::QemuConfig->lock_config($vmid, sub {
6379
6380         $conf = PVE::QemuConfig->load_config($vmid);
6381
6382         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6383         PVE::QemuConfig->check_lock($conf)
6384             if !($skiplock || $is_backing_up);
6385
6386         die "cannot suspend to disk during backup\n"
6387             if $is_backing_up && $includestate;
6388
6389         if ($includestate) {
6390             $conf->{lock} = 'suspending';
6391             my $date = strftime("%Y-%m-%d", localtime(time()));
6392             $storecfg = PVE::Storage::config();
6393             if (!$statestorage) {
6394                 $statestorage = find_vmstate_storage($conf, $storecfg);
6395                 # check permissions for the storage
6396                 my $rpcenv = PVE::RPCEnvironment::get();
6397                 if ($rpcenv->{type} ne 'cli') {
6398                     my $authuser = $rpcenv->get_user();
6399                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6400                 }
6401             }
6402
6403
6404             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6405                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6406             $path = PVE::Storage::path($storecfg, $vmstate);
6407             PVE::QemuConfig->write_config($vmid, $conf);
6408         } else {
6409             mon_cmd($vmid, "stop");
6410         }
6411     });
6412
6413     if ($includestate) {
6414         # save vm state
6415         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6416
6417         eval {
6418             set_migration_caps($vmid, 1);
6419             mon_cmd($vmid, "savevm-start", statefile => $path);
6420             for(;;) {
6421                 my $state = mon_cmd($vmid, "query-savevm");
6422                 if (!$state->{status}) {
6423                     die "savevm not active\n";
6424                 } elsif ($state->{status} eq 'active') {
6425                     sleep(1);
6426                     next;
6427                 } elsif ($state->{status} eq 'completed') {
6428                     print "State saved, quitting\n";
6429                     last;
6430                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6431                     die "query-savevm failed with error '$state->{error}'\n"
6432                 } else {
6433                     die "query-savevm returned status '$state->{status}'\n";
6434                 }
6435             }
6436         };
6437         my $err = $@;
6438
6439         PVE::QemuConfig->lock_config($vmid, sub {
6440             $conf = PVE::QemuConfig->load_config($vmid);
6441             if ($err) {
6442                 # cleanup, but leave suspending lock, to indicate something went wrong
6443                 eval {
6444                     mon_cmd($vmid, "savevm-end");
6445                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6446                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6447                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6448                     PVE::QemuConfig->write_config($vmid, $conf);
6449                 };
6450                 warn $@ if $@;
6451                 die $err;
6452             }
6453
6454             die "lock changed unexpectedly\n"
6455                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6456
6457             mon_cmd($vmid, "quit");
6458             $conf->{lock} = 'suspended';
6459             PVE::QemuConfig->write_config($vmid, $conf);
6460         });
6461     }
6462 }
6463
6464 # $nocheck is set when called as part of a migration - in this context the
6465 # location of the config file (source or target node) is not deterministic,
6466 # since migration cannot wait for pmxcfs to process the rename
6467 sub vm_resume {
6468     my ($vmid, $skiplock, $nocheck) = @_;
6469
6470     PVE::QemuConfig->lock_config($vmid, sub {
6471         my $res = mon_cmd($vmid, 'query-status');
6472         my $resume_cmd = 'cont';
6473         my $reset = 0;
6474         my $conf;
6475         if ($nocheck) {
6476             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6477             if ($@) {
6478                 my $vmlist = PVE::Cluster::get_vmlist();
6479                 if (exists($vmlist->{ids}->{$vmid})) {
6480                     my $node = $vmlist->{ids}->{$vmid}->{node};
6481                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6482                 }
6483                 if (!$conf) {
6484                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6485                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6486                 }
6487             }
6488         } else {
6489             $conf = PVE::QemuConfig->load_config($vmid);
6490         }
6491
6492         if ($res->{status}) {
6493             return if $res->{status} eq 'running'; # job done, go home
6494             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6495             $reset = 1 if $res->{status} eq 'shutdown';
6496         }
6497
6498         if (!$nocheck) {
6499             PVE::QemuConfig->check_lock($conf)
6500                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6501         }
6502
6503         if ($reset) {
6504             # required if a VM shuts down during a backup and we get a resume
6505             # request before the backup finishes for example
6506             mon_cmd($vmid, "system_reset");
6507         }
6508
6509         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6510
6511         mon_cmd($vmid, $resume_cmd);
6512     });
6513 }
6514
6515 sub vm_sendkey {
6516     my ($vmid, $skiplock, $key) = @_;
6517
6518     PVE::QemuConfig->lock_config($vmid, sub {
6519
6520         my $conf = PVE::QemuConfig->load_config($vmid);
6521
6522         # there is no qmp command, so we use the human monitor command
6523         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6524         die $res if $res ne '';
6525     });
6526 }
6527
6528 sub check_bridge_access {
6529     my ($rpcenv, $authuser, $conf) = @_;
6530
6531     return 1 if $authuser eq 'root@pam';
6532
6533     for my $opt (sort keys $conf->%*) {
6534         next if $opt !~ m/^net\d+$/;
6535         my $net = parse_net($conf->{$opt});
6536         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6537         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6538     }
6539     return 1;
6540 };
6541
6542 sub check_mapping_access {
6543     my ($rpcenv, $user, $conf) = @_;
6544
6545     for my $opt (keys $conf->%*) {
6546         if ($opt =~ m/^usb\d+$/) {
6547             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6548             if (my $host = $device->{host}) {
6549                 die "only root can set '$opt' config for real devices\n"
6550                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6551             } elsif ($device->{mapping}) {
6552                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6553             } else {
6554                 die "either 'host' or 'mapping' must be set.\n";
6555             }
6556         } elsif ($opt =~ m/^hostpci\d+$/) {
6557             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6558             if ($device->{host}) {
6559                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6560             } elsif ($device->{mapping}) {
6561                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6562             } else {
6563                 die "either 'host' or 'mapping' must be set.\n";
6564             }
6565        }
6566    }
6567 };
6568
6569 sub check_restore_permissions {
6570     my ($rpcenv, $user, $conf) = @_;
6571
6572     check_bridge_access($rpcenv, $user, $conf);
6573     check_mapping_access($rpcenv, $user, $conf);
6574 }
6575 # vzdump restore implementaion
6576
6577 sub tar_archive_read_firstfile {
6578     my $archive = shift;
6579
6580     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6581
6582     # try to detect archive type first
6583     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6584         die "unable to open file '$archive'\n";
6585     my $firstfile = <$fh>;
6586     kill 15, $pid;
6587     close $fh;
6588
6589     die "ERROR: archive contaions no data\n" if !$firstfile;
6590     chomp $firstfile;
6591
6592     return $firstfile;
6593 }
6594
6595 sub tar_restore_cleanup {
6596     my ($storecfg, $statfile) = @_;
6597
6598     print STDERR "starting cleanup\n";
6599
6600     if (my $fd = IO::File->new($statfile, "r")) {
6601         while (defined(my $line = <$fd>)) {
6602             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6603                 my $volid = $2;
6604                 eval {
6605                     if ($volid =~ m|^/|) {
6606                         unlink $volid || die 'unlink failed\n';
6607                     } else {
6608                         PVE::Storage::vdisk_free($storecfg, $volid);
6609                     }
6610                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6611                 };
6612                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6613             } else {
6614                 print STDERR "unable to parse line in statfile - $line";
6615             }
6616         }
6617         $fd->close();
6618     }
6619 }
6620
6621 sub restore_file_archive {
6622     my ($archive, $vmid, $user, $opts) = @_;
6623
6624     return restore_vma_archive($archive, $vmid, $user, $opts)
6625         if $archive eq '-';
6626
6627     my $info = PVE::Storage::archive_info($archive);
6628     my $format = $opts->{format} // $info->{format};
6629     my $comp = $info->{compression};
6630
6631     # try to detect archive format
6632     if ($format eq 'tar') {
6633         return restore_tar_archive($archive, $vmid, $user, $opts);
6634     } else {
6635         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6636     }
6637 }
6638
6639 # hepler to remove disks that will not be used after restore
6640 my $restore_cleanup_oldconf = sub {
6641     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6642
6643     my $kept_disks = {};
6644
6645     PVE::QemuConfig->foreach_volume($oldconf, sub {
6646         my ($ds, $drive) = @_;
6647
6648         return if drive_is_cdrom($drive, 1);
6649
6650         my $volid = $drive->{file};
6651         return if !$volid || $volid =~ m|^/|;
6652
6653         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6654         return if !$path || !$owner || ($owner != $vmid);
6655
6656         # Note: only delete disk we want to restore
6657         # other volumes will become unused
6658         if ($virtdev_hash->{$ds}) {
6659             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6660             if (my $err = $@) {
6661                 warn $err;
6662             }
6663         } else {
6664             $kept_disks->{$volid} = 1;
6665         }
6666     });
6667
6668     # after the restore we have no snapshots anymore
6669     for my $snapname (keys $oldconf->{snapshots}->%*) {
6670         my $snap = $oldconf->{snapshots}->{$snapname};
6671         if ($snap->{vmstate}) {
6672             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6673             if (my $err = $@) {
6674                 warn $err;
6675             }
6676         }
6677
6678         for my $volid (keys $kept_disks->%*) {
6679             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6680             warn $@ if $@;
6681         }
6682     }
6683 };
6684
6685 # Helper to parse vzdump backup device hints
6686 #
6687 # $rpcenv: Environment, used to ckeck storage permissions
6688 # $user: User ID, to check storage permissions
6689 # $storecfg: Storage configuration
6690 # $fh: the file handle for reading the configuration
6691 # $devinfo: should contain device sizes for all backu-up'ed devices
6692 # $options: backup options (pool, default storage)
6693 #
6694 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6695 my $parse_backup_hints = sub {
6696     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6697
6698     my $check_storage = sub { # assert if an image can be allocate
6699         my ($storeid, $scfg) = @_;
6700         die "Content type 'images' is not available on storage '$storeid'\n"
6701             if !$scfg->{content}->{images};
6702         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6703             if $user ne 'root@pam';
6704     };
6705
6706     my $virtdev_hash = {};
6707     while (defined(my $line = <$fh>)) {
6708         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6709             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6710             die "archive does not contain data for drive '$virtdev'\n"
6711                 if !$devinfo->{$devname};
6712
6713             if (defined($options->{storage})) {
6714                 $storeid = $options->{storage} || 'local';
6715             } elsif (!$storeid) {
6716                 $storeid = 'local';
6717             }
6718             $format = 'raw' if !$format;
6719             $devinfo->{$devname}->{devname} = $devname;
6720             $devinfo->{$devname}->{virtdev} = $virtdev;
6721             $devinfo->{$devname}->{format} = $format;
6722             $devinfo->{$devname}->{storeid} = $storeid;
6723
6724             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6725             $check_storage->($storeid, $scfg); # permission and content type check
6726
6727             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6728         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6729             my $virtdev = $1;
6730             my $drive = parse_drive($virtdev, $2);
6731
6732             if (drive_is_cloudinit($drive)) {
6733                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6734                 $storeid = $options->{storage} if defined ($options->{storage});
6735                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6736                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6737
6738                 $check_storage->($storeid, $scfg); # permission and content type check
6739
6740                 $virtdev_hash->{$virtdev} = {
6741                     format => $format,
6742                     storeid => $storeid,
6743                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6744                     is_cloudinit => 1,
6745                 };
6746             }
6747         }
6748     }
6749
6750     return $virtdev_hash;
6751 };
6752
6753 # Helper to allocate and activate all volumes required for a restore
6754 #
6755 # $storecfg: Storage configuration
6756 # $virtdev_hash: as returned by parse_backup_hints()
6757 #
6758 # Returns: { $virtdev => $volid }
6759 my $restore_allocate_devices = sub {
6760     my ($storecfg, $virtdev_hash, $vmid) = @_;
6761
6762     my $map = {};
6763     foreach my $virtdev (sort keys %$virtdev_hash) {
6764         my $d = $virtdev_hash->{$virtdev};
6765         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6766         my $storeid = $d->{storeid};
6767         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6768
6769         # test if requested format is supported
6770         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6771         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6772         $d->{format} = $defFormat if !$supported;
6773
6774         my $name;
6775         if ($d->{is_cloudinit}) {
6776             $name = "vm-$vmid-cloudinit";
6777             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6778             if ($scfg->{path}) {
6779                 $name .= ".$d->{format}";
6780             }
6781         }
6782
6783         my $volid = PVE::Storage::vdisk_alloc(
6784             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6785
6786         print STDERR "new volume ID is '$volid'\n";
6787         $d->{volid} = $volid;
6788
6789         PVE::Storage::activate_volumes($storecfg, [$volid]);
6790
6791         $map->{$virtdev} = $volid;
6792     }
6793
6794     return $map;
6795 };
6796
6797 sub restore_update_config_line {
6798     my ($cookie, $map, $line, $unique) = @_;
6799
6800     return '' if $line =~ m/^\#qmdump\#/;
6801     return '' if $line =~ m/^\#vzdump\#/;
6802     return '' if $line =~ m/^lock:/;
6803     return '' if $line =~ m/^unused\d+:/;
6804     return '' if $line =~ m/^parent:/;
6805
6806     my $res = '';
6807
6808     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6809     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6810         # try to convert old 1.X settings
6811         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6812         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6813             my ($model, $macaddr) = split(/\=/, $devconfig);
6814             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6815             my $net = {
6816                 model => $model,
6817                 bridge => "vmbr$ind",
6818                 macaddr => $macaddr,
6819             };
6820             my $netstr = print_net($net);
6821
6822             $res .= "net$cookie->{netcount}: $netstr\n";
6823             $cookie->{netcount}++;
6824         }
6825     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6826         my ($id, $netstr) = ($1, $2);
6827         my $net = parse_net($netstr);
6828         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6829         $netstr = print_net($net);
6830         $res .= "$id: $netstr\n";
6831     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6832         my $virtdev = $1;
6833         my $value = $3;
6834         my $di = parse_drive($virtdev, $value);
6835         if (defined($di->{backup}) && !$di->{backup}) {
6836             $res .= "#$line";
6837         } elsif ($map->{$virtdev}) {
6838             delete $di->{format}; # format can change on restore
6839             $di->{file} = $map->{$virtdev};
6840             $value = print_drive($di);
6841             $res .= "$virtdev: $value\n";
6842         } else {
6843             $res .= $line;
6844         }
6845     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6846         my $vmgenid = $1;
6847         if ($vmgenid ne '0') {
6848             # always generate a new vmgenid if there was a valid one setup
6849             $vmgenid = generate_uuid();
6850         }
6851         $res .= "vmgenid: $vmgenid\n";
6852     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6853         my ($uuid, $uuid_str);
6854         UUID::generate($uuid);
6855         UUID::unparse($uuid, $uuid_str);
6856         my $smbios1 = parse_smbios1($2);
6857         $smbios1->{uuid} = $uuid_str;
6858         $res .= $1.print_smbios1($smbios1)."\n";
6859     } else {
6860         $res .= $line;
6861     }
6862
6863     return $res;
6864 }
6865
6866 my $restore_deactivate_volumes = sub {
6867     my ($storecfg, $virtdev_hash) = @_;
6868
6869     my $vollist = [];
6870     for my $dev (values $virtdev_hash->%*) {
6871         push $vollist->@*, $dev->{volid} if $dev->{volid};
6872     }
6873
6874     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6875     print STDERR $@ if $@;
6876 };
6877
6878 my $restore_destroy_volumes = sub {
6879     my ($storecfg, $virtdev_hash) = @_;
6880
6881     for my $dev (values $virtdev_hash->%*) {
6882         my $volid = $dev->{volid} or next;
6883         eval {
6884             PVE::Storage::vdisk_free($storecfg, $volid);
6885             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6886         };
6887         print STDERR "unable to cleanup '$volid' - $@" if $@;
6888     }
6889 };
6890
6891 sub restore_merge_config {
6892     my ($filename, $backup_conf_raw, $override_conf) = @_;
6893
6894     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6895     for my $key (keys $override_conf->%*) {
6896         $backup_conf->{$key} = $override_conf->{$key};
6897     }
6898
6899     return $backup_conf;
6900 }
6901
6902 sub scan_volids {
6903     my ($cfg, $vmid) = @_;
6904
6905     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6906
6907     my $volid_hash = {};
6908     foreach my $storeid (keys %$info) {
6909         foreach my $item (@{$info->{$storeid}}) {
6910             next if !($item->{volid} && $item->{size});
6911             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6912             $volid_hash->{$item->{volid}} = $item;
6913         }
6914     }
6915
6916     return $volid_hash;
6917 }
6918
6919 sub update_disk_config {
6920     my ($vmid, $conf, $volid_hash) = @_;
6921
6922     my $changes;
6923     my $prefix = "VM $vmid";
6924
6925     # used and unused disks
6926     my $referenced = {};
6927
6928     # Note: it is allowed to define multiple storages with same path (alias), so
6929     # we need to check both 'volid' and real 'path' (two different volid can point
6930     # to the same path).
6931
6932     my $referencedpath = {};
6933
6934     # update size info
6935     PVE::QemuConfig->foreach_volume($conf, sub {
6936         my ($opt, $drive) = @_;
6937
6938         my $volid = $drive->{file};
6939         return if !$volid;
6940         my $volume = $volid_hash->{$volid};
6941
6942         # mark volid as "in-use" for next step
6943         $referenced->{$volid} = 1;
6944         if ($volume && (my $path = $volume->{path})) {
6945             $referencedpath->{$path} = 1;
6946         }
6947
6948         return if drive_is_cdrom($drive);
6949         return if !$volume;
6950
6951         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6952         if (defined($updated)) {
6953             $changes = 1;
6954             $conf->{$opt} = print_drive($updated);
6955             print "$prefix ($opt): $msg\n";
6956         }
6957     });
6958
6959     # remove 'unusedX' entry if volume is used
6960     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6961         my ($opt, $drive) = @_;
6962
6963         my $volid = $drive->{file};
6964         return if !$volid;
6965
6966         my $path;
6967         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6968         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6969             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6970             $changes = 1;
6971             delete $conf->{$opt};
6972         }
6973
6974         $referenced->{$volid} = 1;
6975         $referencedpath->{$path} = 1 if $path;
6976     });
6977
6978     foreach my $volid (sort keys %$volid_hash) {
6979         next if $volid =~ m/vm-$vmid-state-/;
6980         next if $referenced->{$volid};
6981         my $path = $volid_hash->{$volid}->{path};
6982         next if !$path; # just to be sure
6983         next if $referencedpath->{$path};
6984         $changes = 1;
6985         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6986         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6987         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6988     }
6989
6990     return $changes;
6991 }
6992
6993 sub rescan {
6994     my ($vmid, $nolock, $dryrun) = @_;
6995
6996     my $cfg = PVE::Storage::config();
6997
6998     print "rescan volumes...\n";
6999     my $volid_hash = scan_volids($cfg, $vmid);
7000
7001     my $updatefn =  sub {
7002         my ($vmid) = @_;
7003
7004         my $conf = PVE::QemuConfig->load_config($vmid);
7005
7006         PVE::QemuConfig->check_lock($conf);
7007
7008         my $vm_volids = {};
7009         foreach my $volid (keys %$volid_hash) {
7010             my $info = $volid_hash->{$volid};
7011             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
7012         }
7013
7014         my $changes = update_disk_config($vmid, $conf, $vm_volids);
7015
7016         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
7017     };
7018
7019     if (defined($vmid)) {
7020         if ($nolock) {
7021             &$updatefn($vmid);
7022         } else {
7023             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
7024         }
7025     } else {
7026         my $vmlist = config_list();
7027         foreach my $vmid (keys %$vmlist) {
7028             if ($nolock) {
7029                 &$updatefn($vmid);
7030             } else {
7031                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
7032             }
7033         }
7034     }
7035 }
7036
7037 sub restore_proxmox_backup_archive {
7038     my ($archive, $vmid, $user, $options) = @_;
7039
7040     my $storecfg = PVE::Storage::config();
7041
7042     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
7043     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7044
7045     my $fingerprint = $scfg->{fingerprint};
7046     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
7047
7048     my $repo = PVE::PBSClient::get_repository($scfg);
7049     my $namespace = $scfg->{namespace};
7050
7051     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7052     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7053     local $ENV{PBS_PASSWORD} = $password;
7054     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7055
7056     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7057         PVE::Storage::parse_volname($storecfg, $archive);
7058
7059     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7060
7061     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7062
7063     my $tmpdir = "/var/tmp/vzdumptmp$$";
7064     rmtree $tmpdir;
7065     mkpath $tmpdir;
7066
7067     my $conffile = PVE::QemuConfig->config_file($vmid);
7068      # disable interrupts (always do cleanups)
7069     local $SIG{INT} =
7070         local $SIG{TERM} =
7071         local $SIG{QUIT} =
7072         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7073
7074     # Note: $oldconf is undef if VM does not exists
7075     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7076     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7077     my $new_conf_raw = '';
7078
7079     my $rpcenv = PVE::RPCEnvironment::get();
7080     my $devinfo = {}; # info about drives included in backup
7081     my $virtdev_hash = {}; # info about allocated drives
7082
7083     eval {
7084         # enable interrupts
7085         local $SIG{INT} =
7086             local $SIG{TERM} =
7087             local $SIG{QUIT} =
7088             local $SIG{HUP} =
7089             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7090
7091         my $cfgfn = "$tmpdir/qemu-server.conf";
7092         my $firewall_config_fn = "$tmpdir/fw.conf";
7093         my $index_fn = "$tmpdir/index.json";
7094
7095         my $cmd = "restore";
7096
7097         my $param = [$pbs_backup_name, "index.json", $index_fn];
7098         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7099         my $index = PVE::Tools::file_get_contents($index_fn);
7100         $index = decode_json($index);
7101
7102         foreach my $info (@{$index->{files}}) {
7103             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7104                 my $devname = $1;
7105                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7106                     $devinfo->{$devname}->{size} = $1;
7107                 } else {
7108                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7109                 }
7110             }
7111         }
7112
7113         my $is_qemu_server_backup = scalar(
7114             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7115         );
7116         if (!$is_qemu_server_backup) {
7117             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7118         }
7119         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7120
7121         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7122         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7123
7124         if ($has_firewall_config) {
7125             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7126             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7127
7128             my $pve_firewall_dir = '/etc/pve/firewall';
7129             mkdir $pve_firewall_dir; # make sure the dir exists
7130             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7131         }
7132
7133         my $fh = IO::File->new($cfgfn, "r") ||
7134             die "unable to read qemu-server.conf - $!\n";
7135
7136         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7137
7138         # fixme: rate limit?
7139
7140         # create empty/temp config
7141         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7142
7143         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7144
7145         # allocate volumes
7146         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7147
7148         foreach my $virtdev (sort keys %$virtdev_hash) {
7149             my $d = $virtdev_hash->{$virtdev};
7150             next if $d->{is_cloudinit}; # no need to restore cloudinit
7151
7152             # this fails if storage is unavailable
7153             my $volid = $d->{volid};
7154             my $path = PVE::Storage::path($storecfg, $volid);
7155
7156             # for live-restore we only want to preload the efidisk and TPM state
7157             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7158
7159             my @ns_arg;
7160             if (defined(my $ns = $scfg->{namespace})) {
7161                 @ns_arg = ('--ns', $ns);
7162             }
7163
7164             my $pbs_restore_cmd = [
7165                 '/usr/bin/pbs-restore',
7166                 '--repository', $repo,
7167                 @ns_arg,
7168                 $pbs_backup_name,
7169                 "$d->{devname}.img.fidx",
7170                 $path,
7171                 '--verbose',
7172                 ];
7173
7174             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7175             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7176
7177             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7178                 push @$pbs_restore_cmd, '--skip-zero';
7179             }
7180
7181             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7182             print "restore proxmox backup image: $dbg_cmdstring\n";
7183             run_command($pbs_restore_cmd);
7184         }
7185
7186         $fh->seek(0, 0) || die "seek failed - $!\n";
7187
7188         my $cookie = { netcount => 0 };
7189         while (defined(my $line = <$fh>)) {
7190             $new_conf_raw .= restore_update_config_line(
7191                 $cookie,
7192                 $map,
7193                 $line,
7194                 $options->{unique},
7195             );
7196         }
7197
7198         $fh->close();
7199     };
7200     my $err = $@;
7201
7202     if ($err || !$options->{live}) {
7203         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7204     }
7205
7206     rmtree $tmpdir;
7207
7208     if ($err) {
7209         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7210         die $err;
7211     }
7212
7213     if ($options->{live}) {
7214         # keep lock during live-restore
7215         $new_conf_raw .= "\nlock: create";
7216     }
7217
7218     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7219     check_restore_permissions($rpcenv, $user, $new_conf);
7220     PVE::QemuConfig->write_config($vmid, $new_conf);
7221
7222     eval { rescan($vmid, 1); };
7223     warn $@ if $@;
7224
7225     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7226
7227     if ($options->{live}) {
7228         # enable interrupts
7229         local $SIG{INT} =
7230             local $SIG{TERM} =
7231             local $SIG{QUIT} =
7232             local $SIG{HUP} =
7233             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7234
7235         my $conf = PVE::QemuConfig->load_config($vmid);
7236         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7237
7238         # these special drives are already restored before start
7239         delete $devinfo->{'drive-efidisk0'};
7240         delete $devinfo->{'drive-tpmstate0-backup'};
7241
7242         my $pbs_opts = {
7243             repo => $repo,
7244             keyfile => $keyfile,
7245             snapshot => $pbs_backup_name,
7246             namespace => $namespace,
7247         };
7248         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7249
7250         PVE::QemuConfig->remove_lock($vmid, "create");
7251     }
7252 }
7253
7254 sub pbs_live_restore {
7255     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7256
7257     print "starting VM for live-restore\n";
7258     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7259
7260     my $pbs_backing = {};
7261     for my $ds (keys %$restored_disks) {
7262         $ds =~ m/^drive-(.*)$/;
7263         my $confname = $1;
7264         $pbs_backing->{$confname} = {
7265             repository => $opts->{repo},
7266             snapshot => $opts->{snapshot},
7267             archive => "$ds.img.fidx",
7268         };
7269         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7270         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7271
7272         my $drive = parse_drive($confname, $conf->{$confname});
7273         print "restoring '$ds' to '$drive->{file}'\n";
7274     }
7275
7276     my $drives_streamed = 0;
7277     eval {
7278         # make sure HA doesn't interrupt our restore by stopping the VM
7279         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7280             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7281         }
7282
7283         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7284         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7285         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7286
7287         my $qmeventd_fd = register_qmeventd_handle($vmid);
7288
7289         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7290         # this will effectively collapse the backing image chain consisting of
7291         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7292         # removes itself once all backing images vanish with 'auto-remove=on')
7293         my $jobs = {};
7294         for my $ds (sort keys %$restored_disks) {
7295             my $job_id = "restore-$ds";
7296             mon_cmd($vmid, 'block-stream',
7297                 'job-id' => $job_id,
7298                 device => "$ds",
7299             );
7300             $jobs->{$job_id} = {};
7301         }
7302
7303         mon_cmd($vmid, 'cont');
7304         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7305
7306         print "restore-drive jobs finished successfully, removing all tracking block devices"
7307             ." to disconnect from Proxmox Backup Server\n";
7308
7309         for my $ds (sort keys %$restored_disks) {
7310             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7311         }
7312
7313         close($qmeventd_fd);
7314     };
7315
7316     my $err = $@;
7317
7318     if ($err) {
7319         warn "An error occurred during live-restore: $err\n";
7320         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7321         die "live-restore failed\n";
7322     }
7323 }
7324
7325 sub restore_vma_archive {
7326     my ($archive, $vmid, $user, $opts, $comp) = @_;
7327
7328     my $readfrom = $archive;
7329
7330     my $cfg = PVE::Storage::config();
7331     my $commands = [];
7332     my $bwlimit = $opts->{bwlimit};
7333
7334     my $dbg_cmdstring = '';
7335     my $add_pipe = sub {
7336         my ($cmd) = @_;
7337         push @$commands, $cmd;
7338         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7339         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7340         $readfrom = '-';
7341     };
7342
7343     my $input = undef;
7344     if ($archive eq '-') {
7345         $input = '<&STDIN';
7346     } else {
7347         # If we use a backup from a PVE defined storage we also consider that
7348         # storage's rate limit:
7349         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7350         if (defined($volid)) {
7351             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7352             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7353             if ($readlimit) {
7354                 print STDERR "applying read rate limit: $readlimit\n";
7355                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7356                 $add_pipe->($cstream);
7357             }
7358         }
7359     }
7360
7361     if ($comp) {
7362         my $info = PVE::Storage::decompressor_info('vma', $comp);
7363         my $cmd = $info->{decompressor};
7364         push @$cmd, $readfrom;
7365         $add_pipe->($cmd);
7366     }
7367
7368     my $tmpdir = "/var/tmp/vzdumptmp$$";
7369     rmtree $tmpdir;
7370
7371     # disable interrupts (always do cleanups)
7372     local $SIG{INT} =
7373         local $SIG{TERM} =
7374         local $SIG{QUIT} =
7375         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7376
7377     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7378     POSIX::mkfifo($mapfifo, 0600);
7379     my $fifofh;
7380     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7381
7382     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7383
7384     my $devinfo = {}; # info about drives included in backup
7385     my $virtdev_hash = {}; # info about allocated drives
7386
7387     my $rpcenv = PVE::RPCEnvironment::get();
7388
7389     my $conffile = PVE::QemuConfig->config_file($vmid);
7390
7391     # Note: $oldconf is undef if VM does not exist
7392     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7393     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7394     my $new_conf_raw = '';
7395
7396     my %storage_limits;
7397
7398     my $print_devmap = sub {
7399         my $cfgfn = "$tmpdir/qemu-server.conf";
7400
7401         # we can read the config - that is already extracted
7402         my $fh = IO::File->new($cfgfn, "r") ||
7403             die "unable to read qemu-server.conf - $!\n";
7404
7405         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7406         if (-f $fwcfgfn) {
7407             my $pve_firewall_dir = '/etc/pve/firewall';
7408             mkdir $pve_firewall_dir; # make sure the dir exists
7409             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7410         }
7411
7412         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7413
7414         foreach my $info (values %{$virtdev_hash}) {
7415             my $storeid = $info->{storeid};
7416             next if defined($storage_limits{$storeid});
7417
7418             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7419             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7420             $storage_limits{$storeid} = $limit * 1024;
7421         }
7422
7423         foreach my $devname (keys %$devinfo) {
7424             die "found no device mapping information for device '$devname'\n"
7425                 if !$devinfo->{$devname}->{virtdev};
7426         }
7427
7428         # create empty/temp config
7429         if ($oldconf) {
7430             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7431             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7432         }
7433
7434         # allocate volumes
7435         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7436
7437         # print restore information to $fifofh
7438         foreach my $virtdev (sort keys %$virtdev_hash) {
7439             my $d = $virtdev_hash->{$virtdev};
7440             next if $d->{is_cloudinit}; # no need to restore cloudinit
7441
7442             my $storeid = $d->{storeid};
7443             my $volid = $d->{volid};
7444
7445             my $map_opts = '';
7446             if (my $limit = $storage_limits{$storeid}) {
7447                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7448             }
7449
7450             my $write_zeros = 1;
7451             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7452                 $write_zeros = 0;
7453             }
7454
7455             my $path = PVE::Storage::path($cfg, $volid);
7456
7457             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7458
7459             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7460         }
7461
7462         $fh->seek(0, 0) || die "seek failed - $!\n";
7463
7464         my $cookie = { netcount => 0 };
7465         while (defined(my $line = <$fh>)) {
7466             $new_conf_raw .= restore_update_config_line(
7467                 $cookie,
7468                 $map,
7469                 $line,
7470                 $opts->{unique},
7471             );
7472         }
7473
7474         $fh->close();
7475     };
7476
7477     my $oldtimeout;
7478
7479     eval {
7480         # enable interrupts
7481         local $SIG{INT} =
7482             local $SIG{TERM} =
7483             local $SIG{QUIT} =
7484             local $SIG{HUP} =
7485             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7486         local $SIG{ALRM} = sub { die "got timeout\n"; };
7487
7488         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7489
7490         my $parser = sub {
7491             my $line = shift;
7492
7493             print "$line\n";
7494
7495             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7496                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7497                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7498             } elsif ($line =~ m/^CTIME: /) {
7499                 # we correctly received the vma config, so we can disable
7500                 # the timeout now for disk allocation
7501                 alarm($oldtimeout || 0);
7502                 $oldtimeout = undef;
7503                 &$print_devmap();
7504                 print $fifofh "done\n";
7505                 close($fifofh);
7506                 $fifofh = undef;
7507             }
7508         };
7509
7510         print "restore vma archive: $dbg_cmdstring\n";
7511         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7512     };
7513     my $err = $@;
7514
7515     alarm($oldtimeout) if $oldtimeout;
7516
7517     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7518
7519     close($fifofh) if $fifofh;
7520     unlink $mapfifo;
7521     rmtree $tmpdir;
7522
7523     if ($err) {
7524         $restore_destroy_volumes->($cfg, $virtdev_hash);
7525         die $err;
7526     }
7527
7528     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7529     check_restore_permissions($rpcenv, $user, $new_conf);
7530     PVE::QemuConfig->write_config($vmid, $new_conf);
7531
7532     eval { rescan($vmid, 1); };
7533     warn $@ if $@;
7534
7535     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7536 }
7537
7538 sub restore_tar_archive {
7539     my ($archive, $vmid, $user, $opts) = @_;
7540
7541     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7542         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7543         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7544     }
7545
7546     if ($archive ne '-') {
7547         my $firstfile = tar_archive_read_firstfile($archive);
7548         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7549             if $firstfile ne 'qemu-server.conf';
7550     }
7551
7552     my $storecfg = PVE::Storage::config();
7553
7554     # avoid zombie disks when restoring over an existing VM -> cleanup first
7555     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7556     # skiplock=1 because qmrestore has set the 'create' lock itself already
7557     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7558     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7559
7560     my $tocmd = "/usr/lib/qemu-server/qmextract";
7561
7562     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7563     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7564     $tocmd .= ' --prealloc' if $opts->{prealloc};
7565     $tocmd .= ' --info' if $opts->{info};
7566
7567     # tar option "xf" does not autodetect compression when read from STDIN,
7568     # so we pipe to zcat
7569     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7570         PVE::Tools::shellquote("--to-command=$tocmd");
7571
7572     my $tmpdir = "/var/tmp/vzdumptmp$$";
7573     mkpath $tmpdir;
7574
7575     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7576     local $ENV{VZDUMP_VMID} = $vmid;
7577     local $ENV{VZDUMP_USER} = $user;
7578
7579     my $conffile = PVE::QemuConfig->config_file($vmid);
7580     my $new_conf_raw = '';
7581
7582     # disable interrupts (always do cleanups)
7583     local $SIG{INT} =
7584         local $SIG{TERM} =
7585         local $SIG{QUIT} =
7586         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7587
7588     eval {
7589         # enable interrupts
7590         local $SIG{INT} =
7591             local $SIG{TERM} =
7592             local $SIG{QUIT} =
7593             local $SIG{HUP} =
7594             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7595
7596         if ($archive eq '-') {
7597             print "extracting archive from STDIN\n";
7598             run_command($cmd, input => "<&STDIN");
7599         } else {
7600             print "extracting archive '$archive'\n";
7601             run_command($cmd);
7602         }
7603
7604         return if $opts->{info};
7605
7606         # read new mapping
7607         my $map = {};
7608         my $statfile = "$tmpdir/qmrestore.stat";
7609         if (my $fd = IO::File->new($statfile, "r")) {
7610             while (defined (my $line = <$fd>)) {
7611                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7612                     $map->{$1} = $2 if $1;
7613                 } else {
7614                     print STDERR "unable to parse line in statfile - $line\n";
7615                 }
7616             }
7617             $fd->close();
7618         }
7619
7620         my $confsrc = "$tmpdir/qemu-server.conf";
7621
7622         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7623
7624         my $cookie = { netcount => 0 };
7625         while (defined (my $line = <$srcfd>)) {
7626             $new_conf_raw .= restore_update_config_line(
7627                 $cookie,
7628                 $map,
7629                 $line,
7630                 $opts->{unique},
7631             );
7632         }
7633
7634         $srcfd->close();
7635     };
7636     if (my $err = $@) {
7637         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7638         die $err;
7639     }
7640
7641     rmtree $tmpdir;
7642
7643     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7644
7645     PVE::Cluster::cfs_update(); # make sure we read new file
7646
7647     eval { rescan($vmid, 1); };
7648     warn $@ if $@;
7649 };
7650
7651 sub foreach_storage_used_by_vm {
7652     my ($conf, $func) = @_;
7653
7654     my $sidhash = {};
7655
7656     PVE::QemuConfig->foreach_volume($conf, sub {
7657         my ($ds, $drive) = @_;
7658         return if drive_is_cdrom($drive);
7659
7660         my $volid = $drive->{file};
7661
7662         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7663         $sidhash->{$sid} = $sid if $sid;
7664     });
7665
7666     foreach my $sid (sort keys %$sidhash) {
7667         &$func($sid);
7668     }
7669 }
7670
7671 my $qemu_snap_storage = {
7672     rbd => 1,
7673 };
7674 sub do_snapshots_with_qemu {
7675     my ($storecfg, $volid, $deviceid) = @_;
7676
7677     return if $deviceid =~ m/tpmstate0/;
7678
7679     my $storage_name = PVE::Storage::parse_volume_id($volid);
7680     my $scfg = $storecfg->{ids}->{$storage_name};
7681     die "could not find storage '$storage_name'\n" if !defined($scfg);
7682
7683     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7684         return 1;
7685     }
7686
7687     if ($volid =~ m/\.(qcow2|qed)$/){
7688         return 1;
7689     }
7690
7691     return;
7692 }
7693
7694 sub qga_check_running {
7695     my ($vmid, $nowarn) = @_;
7696
7697     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7698     if ($@) {
7699         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7700         return 0;
7701     }
7702     return 1;
7703 }
7704
7705 sub template_create {
7706     my ($vmid, $conf, $disk) = @_;
7707
7708     my $storecfg = PVE::Storage::config();
7709
7710     PVE::QemuConfig->foreach_volume($conf, sub {
7711         my ($ds, $drive) = @_;
7712
7713         return if drive_is_cdrom($drive);
7714         return if $disk && $ds ne $disk;
7715
7716         my $volid = $drive->{file};
7717         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7718
7719         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7720         $drive->{file} = $voliddst;
7721         $conf->{$ds} = print_drive($drive);
7722         PVE::QemuConfig->write_config($vmid, $conf);
7723     });
7724 }
7725
7726 sub convert_iscsi_path {
7727     my ($path) = @_;
7728
7729     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7730         my $portal = $1;
7731         my $target = $2;
7732         my $lun = $3;
7733
7734         my $initiator_name = get_initiator_name();
7735
7736         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7737                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7738     }
7739
7740     die "cannot convert iscsi path '$path', unkown format\n";
7741 }
7742
7743 sub qemu_img_convert {
7744     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7745
7746     my $storecfg = PVE::Storage::config();
7747     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7748     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7749
7750     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7751
7752     my $cachemode;
7753     my $src_path;
7754     my $src_is_iscsi = 0;
7755     my $src_format;
7756
7757     if ($src_storeid) {
7758         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7759         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7760         $src_format = qemu_img_format($src_scfg, $src_volname);
7761         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7762         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7763         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7764     } elsif (-f $src_volid || -b $src_volid) {
7765         $src_path = $src_volid;
7766         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7767             $src_format = $1;
7768         }
7769     }
7770
7771     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7772
7773     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7774     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7775     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7776     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7777
7778     my $cmd = [];
7779     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7780     push @$cmd, '-l', "snapshot.name=$snapname"
7781         if $snapname && $src_format && $src_format eq "qcow2";
7782     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7783     push @$cmd, '-T', $cachemode if defined($cachemode);
7784     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7785
7786     if ($src_is_iscsi) {
7787         push @$cmd, '--image-opts';
7788         $src_path = convert_iscsi_path($src_path);
7789     } elsif ($src_format) {
7790         push @$cmd, '-f', $src_format;
7791     }
7792
7793     if ($dst_is_iscsi) {
7794         push @$cmd, '--target-image-opts';
7795         $dst_path = convert_iscsi_path($dst_path);
7796     } else {
7797         push @$cmd, '-O', $dst_format;
7798     }
7799
7800     push @$cmd, $src_path;
7801
7802     if (!$dst_is_iscsi && $is_zero_initialized) {
7803         push @$cmd, "zeroinit:$dst_path";
7804     } else {
7805         push @$cmd, $dst_path;
7806     }
7807
7808     my $parser = sub {
7809         my $line = shift;
7810         if($line =~ m/\((\S+)\/100\%\)/){
7811             my $percent = $1;
7812             my $transferred = int($size * $percent / 100);
7813             my $total_h = render_bytes($size, 1);
7814             my $transferred_h = render_bytes($transferred, 1);
7815
7816             print "transferred $transferred_h of $total_h ($percent%)\n";
7817         }
7818
7819     };
7820
7821     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7822     my $err = $@;
7823     die "copy failed: $err" if $err;
7824 }
7825
7826 sub qemu_img_format {
7827     my ($scfg, $volname) = @_;
7828
7829     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7830         return $1;
7831     } else {
7832         return "raw";
7833     }
7834 }
7835
7836 sub qemu_drive_mirror {
7837     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7838
7839     $jobs = {} if !$jobs;
7840
7841     my $qemu_target;
7842     my $format;
7843     $jobs->{"drive-$drive"} = {};
7844
7845     if ($dst_volid =~ /^nbd:/) {
7846         $qemu_target = $dst_volid;
7847         $format = "nbd";
7848     } else {
7849         my $storecfg = PVE::Storage::config();
7850         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7851
7852         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7853
7854         $format = qemu_img_format($dst_scfg, $dst_volname);
7855
7856         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7857
7858         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7859     }
7860
7861     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7862     $opts->{format} = $format if $format;
7863
7864     if (defined($src_bitmap)) {
7865         $opts->{sync} = 'incremental';
7866         $opts->{bitmap} = $src_bitmap;
7867         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7868     }
7869
7870     if (defined($bwlimit)) {
7871         $opts->{speed} = $bwlimit * 1024;
7872         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7873     } else {
7874         print "drive mirror is starting for drive-$drive\n";
7875     }
7876
7877     # if a job already runs for this device we get an error, catch it for cleanup
7878     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7879     if (my $err = $@) {
7880         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7881         warn "$@\n" if $@;
7882         die "mirroring error: $err\n";
7883     }
7884
7885     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7886 }
7887
7888 # $completion can be either
7889 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7890 # 'cancel': wait until all jobs are ready, block-job-cancel them
7891 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7892 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7893 sub qemu_drive_mirror_monitor {
7894     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7895
7896     $completion //= 'complete';
7897     $op //= "mirror";
7898
7899     eval {
7900         my $err_complete = 0;
7901
7902         my $starttime = time ();
7903         while (1) {
7904             die "block job ('$op') timed out\n" if $err_complete > 300;
7905
7906             my $stats = mon_cmd($vmid, "query-block-jobs");
7907             my $ctime = time();
7908
7909             my $running_jobs = {};
7910             for my $stat (@$stats) {
7911                 next if $stat->{type} ne $op;
7912                 $running_jobs->{$stat->{device}} = $stat;
7913             }
7914
7915             my $readycounter = 0;
7916
7917             for my $job_id (sort keys %$jobs) {
7918                 my $job = $running_jobs->{$job_id};
7919
7920                 my $vanished = !defined($job);
7921                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7922                 if($complete || ($vanished && $completion eq 'auto')) {
7923                     print "$job_id: $op-job finished\n";
7924                     delete $jobs->{$job_id};
7925                     next;
7926                 }
7927
7928                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7929
7930                 my $busy = $job->{busy};
7931                 my $ready = $job->{ready};
7932                 if (my $total = $job->{len}) {
7933                     my $transferred = $job->{offset} || 0;
7934                     my $remaining = $total - $transferred;
7935                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7936
7937                     my $duration = $ctime - $starttime;
7938                     my $total_h = render_bytes($total, 1);
7939                     my $transferred_h = render_bytes($transferred, 1);
7940
7941                     my $status = sprintf(
7942                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7943                         render_duration($duration),
7944                     );
7945
7946                     if ($ready) {
7947                         if ($busy) {
7948                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7949                         } else {
7950                             $status .= ", ready";
7951                         }
7952                     }
7953                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7954                     $jobs->{$job_id}->{ready} = $ready;
7955                 }
7956
7957                 $readycounter++ if $job->{ready};
7958             }
7959
7960             last if scalar(keys %$jobs) == 0;
7961
7962             if ($readycounter == scalar(keys %$jobs)) {
7963                 print "all '$op' jobs are ready\n";
7964
7965                 # do the complete later (or has already been done)
7966                 last if $completion eq 'skip' || $completion eq 'auto';
7967
7968                 if ($vmiddst && $vmiddst != $vmid) {
7969                     my $agent_running = $qga && qga_check_running($vmid);
7970                     if ($agent_running) {
7971                         print "freeze filesystem\n";
7972                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7973                         warn $@ if $@;
7974                     } else {
7975                         print "suspend vm\n";
7976                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7977                         warn $@ if $@;
7978                     }
7979
7980                     # if we clone a disk for a new target vm, we don't switch the disk
7981                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7982
7983                     if ($agent_running) {
7984                         print "unfreeze filesystem\n";
7985                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7986                         warn $@ if $@;
7987                     } else {
7988                         print "resume vm\n";
7989                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7990                         warn $@ if $@;
7991                     }
7992
7993                     last;
7994                 } else {
7995
7996                     for my $job_id (sort keys %$jobs) {
7997                         # try to switch the disk if source and destination are on the same guest
7998                         print "$job_id: Completing block job_id...\n";
7999
8000                         my $op;
8001                         if ($completion eq 'complete') {
8002                             $op = 'block-job-complete';
8003                         } elsif ($completion eq 'cancel') {
8004                             $op = 'block-job-cancel';
8005                         } else {
8006                             die "invalid completion value: $completion\n";
8007                         }
8008                         eval { mon_cmd($vmid, $op, device => $job_id) };
8009                         if ($@ =~ m/cannot be completed/) {
8010                             print "$job_id: block job cannot be completed, trying again.\n";
8011                             $err_complete++;
8012                         }else {
8013                             print "$job_id: Completed successfully.\n";
8014                             $jobs->{$job_id}->{complete} = 1;
8015                         }
8016                     }
8017                 }
8018             }
8019             sleep 1;
8020         }
8021     };
8022     my $err = $@;
8023
8024     if ($err) {
8025         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
8026         die "block job ($op) error: $err";
8027     }
8028 }
8029
8030 sub qemu_blockjobs_cancel {
8031     my ($vmid, $jobs) = @_;
8032
8033     foreach my $job (keys %$jobs) {
8034         print "$job: Cancelling block job\n";
8035         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
8036         $jobs->{$job}->{cancel} = 1;
8037     }
8038
8039     while (1) {
8040         my $stats = mon_cmd($vmid, "query-block-jobs");
8041
8042         my $running_jobs = {};
8043         foreach my $stat (@$stats) {
8044             $running_jobs->{$stat->{device}} = $stat;
8045         }
8046
8047         foreach my $job (keys %$jobs) {
8048
8049             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
8050                 print "$job: Done.\n";
8051                 delete $jobs->{$job};
8052             }
8053         }
8054
8055         last if scalar(keys %$jobs) == 0;
8056
8057         sleep 1;
8058     }
8059 }
8060
8061 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8062 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8063 my sub clone_disk_check_io_uring {
8064     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8065
8066     return if !$use_drive_mirror;
8067
8068     # Don't complain when not changing storage.
8069     # Assume if it works for the source, it'll work for the target too.
8070     return if $src_storeid eq $dst_storeid;
8071
8072     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8073     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8074
8075     my $cache_direct = drive_uses_cache_direct($src_drive);
8076
8077     my $src_uses_io_uring;
8078     if ($src_drive->{aio}) {
8079         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8080     } else {
8081         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8082     }
8083
8084     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8085         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8086 }
8087
8088 sub clone_disk {
8089     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8090
8091     my ($vmid, $running) = $source->@{qw(vmid running)};
8092     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8093
8094     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8095     my ($storage, $format) = $dest->@{qw(storage format)};
8096
8097     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8098
8099     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8100         die "cloning from/to EFI disk requires EFI disk\n"
8101             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8102         die "cloning from/to TPM state requires TPM state\n"
8103             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8104
8105         # This would lead to two device nodes in QEMU pointing to the same backing image!
8106         die "cannot change drive name when cloning disk from/to the same VM\n"
8107             if $use_drive_mirror && $vmid == $newvmid;
8108     }
8109
8110     die "cannot move TPM state while VM is running\n"
8111         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8112
8113     my $newvolid;
8114
8115     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8116     print "$src_drivename " if $src_drivename;
8117     print "($drive->{file})\n";
8118
8119     if (!$full) {
8120         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8121         push @$newvollist, $newvolid;
8122     } else {
8123         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8124         my $storeid = $storage || $src_storeid;
8125
8126         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8127
8128         my $name = undef;
8129         my $size = undef;
8130         if (drive_is_cloudinit($drive)) {
8131             $name = "vm-$newvmid-cloudinit";
8132             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8133             if ($scfg->{path}) {
8134                 $name .= ".$dst_format";
8135             }
8136             $snapname = undef;
8137             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8138         } elsif ($dst_drivename eq 'efidisk0') {
8139             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8140         } elsif ($dst_drivename eq 'tpmstate0') {
8141             $dst_format = 'raw';
8142             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8143         } else {
8144             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8145
8146             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8147         }
8148         $newvolid = PVE::Storage::vdisk_alloc(
8149             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8150         );
8151         push @$newvollist, $newvolid;
8152
8153         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8154
8155         if (drive_is_cloudinit($drive)) {
8156             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8157             # if this is the case, we have to complete any block-jobs still there from
8158             # previous drive-mirrors
8159             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8160                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8161             }
8162             goto no_data_clone;
8163         }
8164
8165         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8166         if ($use_drive_mirror) {
8167             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8168                 $completion, $qga, $bwlimit);
8169         } else {
8170             if ($dst_drivename eq 'efidisk0') {
8171                 # the relevant data on the efidisk may be smaller than the source
8172                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8173                 # that is given by the OVMF_VARS.fd
8174                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8175                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8176
8177                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8178
8179                 # better for Ceph if block size is not too small, see bug #3324
8180                 my $bs = 1024*1024;
8181
8182                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8183
8184                 if ($src_format eq 'qcow2' && $snapname) {
8185                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8186                         if !min_version(kvm_user_version(), 6, 2);
8187                     push $cmd->@*, '-l', $snapname;
8188                 }
8189                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8190                 run_command($cmd);
8191             } else {
8192                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8193             }
8194         }
8195     }
8196
8197 no_data_clone:
8198     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8199
8200     my $disk = dclone($drive);
8201     delete $disk->{format};
8202     $disk->{file} = $newvolid;
8203     $disk->{size} = $size if defined($size);
8204
8205     return $disk;
8206 }
8207
8208 sub get_running_qemu_version {
8209     my ($vmid) = @_;
8210     my $res = mon_cmd($vmid, "query-version");
8211     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8212 }
8213
8214 sub qemu_use_old_bios_files {
8215     my ($machine_type) = @_;
8216
8217     return if !$machine_type;
8218
8219     my $use_old_bios_files = undef;
8220
8221     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8222         $machine_type = $1;
8223         $use_old_bios_files = 1;
8224     } else {
8225         my $version = extract_version($machine_type, kvm_user_version());
8226         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8227         # load new efi bios files on migration. So this hack is required to allow
8228         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8229         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8230         $use_old_bios_files = !min_version($version, 2, 4);
8231     }
8232
8233     return ($use_old_bios_files, $machine_type);
8234 }
8235
8236 sub get_efivars_size {
8237     my ($conf, $efidisk) = @_;
8238
8239     my $arch = get_vm_arch($conf);
8240     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8241     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8242     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8243     return -s $ovmf_vars;
8244 }
8245
8246 sub update_efidisk_size {
8247     my ($conf) = @_;
8248
8249     return if !defined($conf->{efidisk0});
8250
8251     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8252     $disk->{size} = get_efivars_size($conf);
8253     $conf->{efidisk0} = print_drive($disk);
8254
8255     return;
8256 }
8257
8258 sub update_tpmstate_size {
8259     my ($conf) = @_;
8260
8261     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8262     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8263     $conf->{tpmstate0} = print_drive($disk);
8264 }
8265
8266 sub create_efidisk($$$$$$$) {
8267     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8268
8269     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8270
8271     my $vars_size_b = -s $ovmf_vars;
8272     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8273     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8274     PVE::Storage::activate_volumes($storecfg, [$volid]);
8275
8276     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8277     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8278
8279     return ($volid, $size/1024);
8280 }
8281
8282 sub vm_iothreads_list {
8283     my ($vmid) = @_;
8284
8285     my $res = mon_cmd($vmid, 'query-iothreads');
8286
8287     my $iothreads = {};
8288     foreach my $iothread (@$res) {
8289         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8290     }
8291
8292     return $iothreads;
8293 }
8294
8295 sub scsihw_infos {
8296     my ($conf, $drive) = @_;
8297
8298     my $maxdev = 0;
8299
8300     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8301         $maxdev = 7;
8302     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8303         $maxdev = 1;
8304     } else {
8305         $maxdev = 256;
8306     }
8307
8308     my $controller = int($drive->{index} / $maxdev);
8309     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8310         ? "virtioscsi"
8311         : "scsihw";
8312
8313     return ($maxdev, $controller, $controller_prefix);
8314 }
8315
8316 sub resolve_dst_disk_format {
8317         my ($storecfg, $storeid, $src_volname, $format) = @_;
8318         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8319
8320         if (!$format) {
8321             # if no target format is specified, use the source disk format as hint
8322             if ($src_volname) {
8323                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8324                 $format = qemu_img_format($scfg, $src_volname);
8325             } else {
8326                 return $defFormat;
8327             }
8328         }
8329
8330         # test if requested format is supported - else use default
8331         my $supported = grep { $_ eq $format } @$validFormats;
8332         $format = $defFormat if !$supported;
8333         return $format;
8334 }
8335
8336 # NOTE: if this logic changes, please update docs & possibly gui logic
8337 sub find_vmstate_storage {
8338     my ($conf, $storecfg) = @_;
8339
8340     # first, return storage from conf if set
8341     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8342
8343     my ($target, $shared, $local);
8344
8345     foreach_storage_used_by_vm($conf, sub {
8346         my ($sid) = @_;
8347         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8348         my $dst = $scfg->{shared} ? \$shared : \$local;
8349         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8350     });
8351
8352     # second, use shared storage where VM has at least one disk
8353     # third, use local storage where VM has at least one disk
8354     # fall back to local storage
8355     $target = $shared // $local // 'local';
8356
8357     return $target;
8358 }
8359
8360 sub generate_uuid {
8361     my ($uuid, $uuid_str);
8362     UUID::generate($uuid);
8363     UUID::unparse($uuid, $uuid_str);
8364     return $uuid_str;
8365 }
8366
8367 sub generate_smbios1_uuid {
8368     return "uuid=".generate_uuid();
8369 }
8370
8371 sub nbd_stop {
8372     my ($vmid) = @_;
8373
8374     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8375 }
8376
8377 sub create_reboot_request {
8378     my ($vmid) = @_;
8379     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8380         or die "failed to create reboot trigger file: $!\n";
8381     close($fh);
8382 }
8383
8384 sub clear_reboot_request {
8385     my ($vmid) = @_;
8386     my $path = "/run/qemu-server/$vmid.reboot";
8387     my $res = 0;
8388
8389     $res = unlink($path);
8390     die "could not remove reboot request for $vmid: $!"
8391         if !$res && $! != POSIX::ENOENT;
8392
8393     return $res;
8394 }
8395
8396 sub bootorder_from_legacy {
8397     my ($conf, $bootcfg) = @_;
8398
8399     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8400     my $bootindex_hash = {};
8401     my $i = 1;
8402     foreach my $o (split(//, $boot)) {
8403         $bootindex_hash->{$o} = $i*100;
8404         $i++;
8405     }
8406
8407     my $bootorder = {};
8408
8409     PVE::QemuConfig->foreach_volume($conf, sub {
8410         my ($ds, $drive) = @_;
8411
8412         if (drive_is_cdrom ($drive, 1)) {
8413             if ($bootindex_hash->{d}) {
8414                 $bootorder->{$ds} = $bootindex_hash->{d};
8415                 $bootindex_hash->{d} += 1;
8416             }
8417         } elsif ($bootindex_hash->{c}) {
8418             $bootorder->{$ds} = $bootindex_hash->{c}
8419                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8420             $bootindex_hash->{c} += 1;
8421         }
8422     });
8423
8424     if ($bootindex_hash->{n}) {
8425         for (my $i = 0; $i < $MAX_NETS; $i++) {
8426             my $netname = "net$i";
8427             next if !$conf->{$netname};
8428             $bootorder->{$netname} = $bootindex_hash->{n};
8429             $bootindex_hash->{n} += 1;
8430         }
8431     }
8432
8433     return $bootorder;
8434 }
8435
8436 # Generate default device list for 'boot: order=' property. Matches legacy
8437 # default boot order, but with explicit device names. This is important, since
8438 # the fallback for when neither 'order' nor the old format is specified relies
8439 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8440 sub get_default_bootdevices {
8441     my ($conf) = @_;
8442
8443     my @ret = ();
8444
8445     # harddisk
8446     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8447     push @ret, $first if $first;
8448
8449     # cdrom
8450     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8451     push @ret, $first if $first;
8452
8453     # network
8454     for (my $i = 0; $i < $MAX_NETS; $i++) {
8455         my $netname = "net$i";
8456         next if !$conf->{$netname};
8457         push @ret, $netname;
8458         last;
8459     }
8460
8461     return \@ret;
8462 }
8463
8464 sub device_bootorder {
8465     my ($conf) = @_;
8466
8467     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8468
8469     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8470
8471     my $bootorder = {};
8472     if (!defined($boot) || $boot->{legacy}) {
8473         $bootorder = bootorder_from_legacy($conf, $boot);
8474     } elsif ($boot->{order}) {
8475         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8476         for my $dev (PVE::Tools::split_list($boot->{order})) {
8477             $bootorder->{$dev} = $i++;
8478         }
8479     }
8480
8481     return $bootorder;
8482 }
8483
8484 sub register_qmeventd_handle {
8485     my ($vmid) = @_;
8486
8487     my $fh;
8488     my $peer = "/var/run/qmeventd.sock";
8489     my $count = 0;
8490
8491     for (;;) {
8492         $count++;
8493         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8494         last if $fh;
8495         if ($! != EINTR && $! != EAGAIN) {
8496             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8497         }
8498         if ($count > 4) {
8499             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8500               . "after $count retries\n";
8501         }
8502         usleep(25000);
8503     }
8504
8505     # send handshake to mark VM as backing up
8506     print $fh to_json({vzdump => {vmid => "$vmid"}});
8507
8508     # return handle to be closed later when inhibit is no longer required
8509     return $fh;
8510 }
8511
8512 # bash completion helper
8513
8514 sub complete_backup_archives {
8515     my ($cmdname, $pname, $cvalue) = @_;
8516
8517     my $cfg = PVE::Storage::config();
8518
8519     my $storeid;
8520
8521     if ($cvalue =~ m/^([^:]+):/) {
8522         $storeid = $1;
8523     }
8524
8525     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8526
8527     my $res = [];
8528     foreach my $id (keys %$data) {
8529         foreach my $item (@{$data->{$id}}) {
8530             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8531             push @$res, $item->{volid} if defined($item->{volid});
8532         }
8533     }
8534
8535     return $res;
8536 }
8537
8538 my $complete_vmid_full = sub {
8539     my ($running) = @_;
8540
8541     my $idlist = vmstatus();
8542
8543     my $res = [];
8544
8545     foreach my $id (keys %$idlist) {
8546         my $d = $idlist->{$id};
8547         if (defined($running)) {
8548             next if $d->{template};
8549             next if $running && $d->{status} ne 'running';
8550             next if !$running && $d->{status} eq 'running';
8551         }
8552         push @$res, $id;
8553
8554     }
8555     return $res;
8556 };
8557
8558 sub complete_vmid {
8559     return &$complete_vmid_full();
8560 }
8561
8562 sub complete_vmid_stopped {
8563     return &$complete_vmid_full(0);
8564 }
8565
8566 sub complete_vmid_running {
8567     return &$complete_vmid_full(1);
8568 }
8569
8570 sub complete_storage {
8571
8572     my $cfg = PVE::Storage::config();
8573     my $ids = $cfg->{ids};
8574
8575     my $res = [];
8576     foreach my $sid (keys %$ids) {
8577         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8578         next if !$ids->{$sid}->{content}->{images};
8579         push @$res, $sid;
8580     }
8581
8582     return $res;
8583 }
8584
8585 sub complete_migration_storage {
8586     my ($cmd, $param, $current_value, $all_args) = @_;
8587
8588     my $targetnode = @$all_args[1];
8589
8590     my $cfg = PVE::Storage::config();
8591     my $ids = $cfg->{ids};
8592
8593     my $res = [];
8594     foreach my $sid (keys %$ids) {
8595         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8596         next if !$ids->{$sid}->{content}->{images};
8597         push @$res, $sid;
8598     }
8599
8600     return $res;
8601 }
8602
8603 sub vm_is_paused {
8604     my ($vmid, $include_suspended) = @_;
8605     my $qmpstatus = eval {
8606         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8607         mon_cmd($vmid, "query-status");
8608     };
8609     warn "$@\n" if $@;
8610     return $qmpstatus && (
8611         $qmpstatus->{status} eq "paused" ||
8612         $qmpstatus->{status} eq "prelaunch" ||
8613         ($include_suspended && $qmpstatus->{status} eq "suspended")
8614     );
8615 }
8616
8617 sub check_volume_storage_type {
8618     my ($storecfg, $vol) = @_;
8619
8620     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8621     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8622     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8623
8624     die "storage '$storeid' does not support content-type '$vtype'\n"
8625         if !$scfg->{content}->{$vtype};
8626
8627     return 1;
8628 }
8629
8630 sub add_nets_bridge_fdb {
8631     my ($conf, $vmid) = @_;
8632
8633     for my $opt (keys %$conf) {
8634         next if $opt !~ m/^net(\d+)$/;
8635         my $iface = "tap${vmid}i$1";
8636         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8637         my $net = parse_net($conf->{$opt}, 1) or next;
8638
8639         my $mac = $net->{macaddr};
8640         if (!$mac) {
8641             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8642                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8643             next;
8644         }
8645
8646         my $bridge = $net->{bridge};
8647         if (!$bridge) {
8648             log_warn("Interface '$iface' not attached to any bridge.");
8649             next;
8650         }
8651         if ($have_sdn) {
8652             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8653         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8654             PVE::Network::add_bridge_fdb($iface, $mac);
8655         }
8656     }
8657 }
8658
8659 sub del_nets_bridge_fdb {
8660     my ($conf, $vmid) = @_;
8661
8662     for my $opt (keys %$conf) {
8663         next if $opt !~ m/^net(\d+)$/;
8664         my $iface = "tap${vmid}i$1";
8665
8666         my $net = parse_net($conf->{$opt}) or next;
8667         my $mac = $net->{macaddr} or next;
8668
8669         my $bridge = $net->{bridge};
8670         if ($have_sdn) {
8671             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8672         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8673             PVE::Network::del_bridge_fdb($iface, $mac);
8674         }
8675     }
8676 }
8677
8678 sub create_ifaces_ipams_ips {
8679     my ($conf, $vmid) = @_;
8680
8681     return if !$have_sdn;
8682
8683     foreach my $opt (keys %$conf) {
8684         if ($opt =~ m/^net(\d+)$/) {
8685             my $value = $conf->{$opt};
8686             my $net = PVE::QemuServer::parse_net($value);
8687             eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8688             warn $@ if $@;
8689         }
8690     }
8691 }
8692
8693 sub delete_ifaces_ipams_ips {
8694     my ($conf, $vmid) = @_;
8695
8696     return if !$have_sdn;
8697
8698     foreach my $opt (keys %$conf) {
8699         if ($opt =~ m/^net(\d+)$/) {
8700             my $net = PVE::QemuServer::parse_net($conf->{$opt});
8701             eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8702             warn $@ if $@;
8703         }
8704     }
8705 }
8706
8707 1;