PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory qw(get_current_memory);
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  62 use PVE::QemuServer::USB;
  63
  64 my $have_sdn;
  65 eval {
  66     require PVE::Network::SDN::Zones;
  67     require PVE::Network::SDN::Vnets;
  68     $have_sdn = 1;
  69 };
  70
  71 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  72 my $OVMF = {
  73     x86_64 => {
  74         '4m-no-smm' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  77         ],
  78         '4m-no-smm-ms' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  81         ],
  82         '4m' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  85         ],
  86         '4m-ms' => [
  87             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  89         ],
  90         # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
  91         # anymore. how can we deperacate this sanely without breaking existing instances, or using
  92         # older backups and snapshot?
  93         default => [
  94             "$EDK2_FW_BASE/OVMF_CODE.fd",
  95             "$EDK2_FW_BASE/OVMF_VARS.fd",
  96         ],
  97     },
  98     aarch64 => {
  99         default => [
 100             "$EDK2_FW_BASE/AAVMF_CODE.fd",
 101             "$EDK2_FW_BASE/AAVMF_VARS.fd",
 102         ],
 103     },
 104 };
 105
 106 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 107
 108 # Note about locking: we use flock on the config file protect against concurent actions.
 109 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 110 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 111 # But you can ignore this kind of lock with the --skiplock flag.
 112
 113 cfs_register_file(
 114     '/qemu-server/',
 115     \&parse_vm_config,
 116     \&write_vm_config
 117 );
 118
 119 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 120     description => "Some command save/restore state from this location.",
 121     type => 'string',
 122     maxLength => 128,
 123     optional => 1,
 124 });
 125
 126 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 127         description => "Specifies the QEMU machine type.",
 128         type => 'string',
 129         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 130         maxLength => 40,
 131         optional => 1,
 132 });
 133
 134 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 135 my $nodename_cache;
 136 sub nodename {
 137     $nodename_cache //= PVE::INotify::nodename();
 138     return $nodename_cache;
 139 }
 140
 141 my $watchdog_fmt = {
 142     model => {
 143         default_key => 1,
 144         type => 'string',
 145         enum => [qw(i6300esb ib700)],
 146         description => "Watchdog type to emulate.",
 147         default => 'i6300esb',
 148         optional => 1,
 149     },
 150     action => {
 151         type => 'string',
 152         enum => [qw(reset shutdown poweroff pause debug none)],
 153         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 154         optional => 1,
 155     },
 156 };
 157 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 158
 159 my $agent_fmt = {
 160     enabled => {
 161         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 162         type => 'boolean',
 163         default => 0,
 164         default_key => 1,
 165     },
 166     fstrim_cloned_disks => {
 167         description => "Run fstrim after moving a disk or migrating the VM.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0,
 171     },
 172     'freeze-fs-on-backup' => {
 173         description => "Freeze/thaw guest filesystems on backup for consistency.",
 174         type => 'boolean',
 175         optional => 1,
 176         default => 1,
 177     },
 178     type => {
 179         description => "Select the agent type",
 180         type => 'string',
 181         default => 'virtio',
 182         optional => 1,
 183         enum => [qw(virtio isa)],
 184     },
 185 };
 186
 187 my $vga_fmt = {
 188     type => {
 189         description => "Select the VGA type.",
 190         type => 'string',
 191         default => 'std',
 192         optional => 1,
 193         default_key => 1,
 194         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 195     },
 196     memory => {
 197         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 198         type => 'integer',
 199         optional => 1,
 200         minimum => 4,
 201         maximum => 512,
 202     },
 203     clipboard => {
 204         description => 'Enable a specific clipboard. If not set, depending on the display type the'
 205             .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
 206         type => 'string',
 207         enum => ['vnc'],
 208         optional => 1,
 209     },
 210 };
 211
 212 my $ivshmem_fmt = {
 213     size => {
 214         type => 'integer',
 215         minimum => 1,
 216         description => "The size of the file in MB.",
 217     },
 218     name => {
 219         type => 'string',
 220         pattern => '[a-zA-Z0-9\-]+',
 221         optional => 1,
 222         format_description => 'string',
 223         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 224     },
 225 };
 226
 227 my $audio_fmt = {
 228     device => {
 229         type => 'string',
 230         enum => [qw(ich9-intel-hda intel-hda AC97)],
 231         description =>  "Configure an audio device."
 232     },
 233     driver =>  {
 234         type => 'string',
 235         enum => ['spice', 'none'],
 236         default => 'spice',
 237         optional => 1,
 238         description => "Driver backend for the audio device."
 239     },
 240 };
 241
 242 my $spice_enhancements_fmt = {
 243     foldersharing => {
 244         type => 'boolean',
 245         optional => 1,
 246         default => '0',
 247         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 248     },
 249     videostreaming =>  {
 250         type => 'string',
 251         enum => ['off', 'all', 'filter'],
 252         default => 'off',
 253         optional => 1,
 254         description => "Enable video streaming. Uses compression for detected video streams."
 255     },
 256 };
 257
 258 my $rng_fmt = {
 259     source => {
 260         type => 'string',
 261         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 262         default_key => 1,
 263         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 264             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 265             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 266             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 267             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 268             ." a hardware RNG from the host.",
 269     },
 270     max_bytes => {
 271         type => 'integer',
 272         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 273             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 274             ." `0` to disable limiting (potentially dangerous!).",
 275         optional => 1,
 276
 277         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 278         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 279         # reading from /dev/urandom
 280         default => 1024,
 281     },
 282     period => {
 283         type => 'integer',
 284         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 285             ." the guest to retrieve another 'max_bytes' of entropy.",
 286         optional => 1,
 287         default => 1000,
 288     },
 289 };
 290
 291 my $meta_info_fmt = {
 292     'ctime' => {
 293         type => 'integer',
 294         description => "The guest creation timestamp as UNIX epoch time",
 295         minimum => 0,
 296         optional => 1,
 297     },
 298     'creation-qemu' => {
 299         type => 'string',
 300         description => "The QEMU (machine) version from the time this VM was created.",
 301         pattern => '\d+(\.\d+)+',
 302         optional => 1,
 303     },
 304 };
 305
 306 my $confdesc = {
 307     onboot => {
 308         optional => 1,
 309         type => 'boolean',
 310         description => "Specifies whether a VM will be started during system bootup.",
 311         default => 0,
 312     },
 313     autostart => {
 314         optional => 1,
 315         type => 'boolean',
 316         description => "Automatic restart after crash (currently ignored).",
 317         default => 0,
 318     },
 319     hotplug => {
 320         optional => 1,
 321         type => 'string', format => 'pve-hotplug-features',
 322         description => "Selectively enable hotplug features. This is a comma separated list of"
 323             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 324             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 325             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 326             ." windows > 7.",
 327         default => 'network,disk,usb',
 328     },
 329     reboot => {
 330         optional => 1,
 331         type => 'boolean',
 332         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 333         default => 1,
 334     },
 335     lock => {
 336         optional => 1,
 337         type => 'string',
 338         description => "Lock/unlock the VM.",
 339         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 340     },
 341     cpulimit => {
 342         optional => 1,
 343         type => 'number',
 344         description => "Limit of CPU usage.",
 345         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 346             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 347         minimum => 0,
 348         maximum => 128,
 349         default => 0,
 350     },
 351     cpuunits => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 355         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 356             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 357             ." weights of all the other running VMs.",
 358         minimum => 1,
 359         maximum => 262144,
 360         default => 'cgroup v1: 1024, cgroup v2: 100',
 361     },
 362     memory => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Memory properties.",
 366         format => $PVE::QemuServer::Memory::memory_fmt
 367     },
 368     balloon => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 372         minimum => 0,
 373     },
 374     shares => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 378             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 379             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 380         minimum => 0,
 381         maximum => 50000,
 382         default => 1000,
 383     },
 384     keyboard => {
 385         optional => 1,
 386         type => 'string',
 387         description => "Keyboard layout for VNC server. This option is generally not required and"
 388         ." is often better handled from within the guest OS.",
 389         enum => PVE::Tools::kvmkeymaplist(),
 390         default => undef,
 391     },
 392     name => {
 393         optional => 1,
 394         type => 'string', format => 'dns-name',
 395         description => "Set a name for the VM. Only used on the configuration web interface.",
 396     },
 397     scsihw => {
 398         optional => 1,
 399         type => 'string',
 400         description => "SCSI controller model",
 401         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 402         default => 'lsi',
 403     },
 404     description => {
 405         optional => 1,
 406         type => 'string',
 407         description => "Description for the VM. Shown in the web-interface VM's summary."
 408             ." This is saved as comment inside the configuration file.",
 409         maxLength => 1024 * 8,
 410     },
 411     ostype => {
 412         optional => 1,
 413         type => 'string',
 414         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 415         description => "Specify guest operating system.",
 416         verbose_description => <<EODESC,
 417 Specify guest operating system. This is used to enable special
 418 optimization/features for specific operating systems:
 419
 420 [horizontal]
 421 other;; unspecified OS
 422 wxp;; Microsoft Windows XP
 423 w2k;; Microsoft Windows 2000
 424 w2k3;; Microsoft Windows 2003
 425 w2k8;; Microsoft Windows 2008
 426 wvista;; Microsoft Windows Vista
 427 win7;; Microsoft Windows 7
 428 win8;; Microsoft Windows 8/2012/2012r2
 429 win10;; Microsoft Windows 10/2016/2019
 430 win11;; Microsoft Windows 11/2022
 431 l24;; Linux 2.4 Kernel
 432 l26;; Linux 2.6 - 6.X Kernel
 433 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 434 EODESC
 435     },
 436     boot => {
 437         optional => 1,
 438         type => 'string', format => 'pve-qm-boot',
 439         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 440             ." key or 'legacy=' is deprecated.",
 441     },
 442     bootdisk => {
 443         optional => 1,
 444         type => 'string', format => 'pve-qm-bootdisk',
 445         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 446         pattern => '(ide|sata|scsi|virtio)\d+',
 447     },
 448     smp => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "The number of CPUs. Please use option -sockets instead.",
 452         minimum => 1,
 453         default => 1,
 454     },
 455     sockets => {
 456         optional => 1,
 457         type => 'integer',
 458         description => "The number of CPU sockets.",
 459         minimum => 1,
 460         default => 1,
 461     },
 462     cores => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "The number of cores per socket.",
 466         minimum => 1,
 467         default => 1,
 468     },
 469     numa => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable NUMA.",
 473         default => 0,
 474     },
 475     hugepages => {
 476         optional => 1,
 477         type => 'string',
 478         description => "Enable/disable hugepages memory.",
 479         enum => [qw(any 2 1024)],
 480     },
 481     keephugepages => {
 482         optional => 1,
 483         type => 'boolean',
 484         default => 0,
 485         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 486             ." after VM shutdown and can be used for subsequent starts.",
 487     },
 488     vcpus => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Number of hotplugged vcpus.",
 492         minimum => 1,
 493         default => 0,
 494     },
 495     acpi => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable ACPI.",
 499         default => 1,
 500     },
 501     agent => {
 502         optional => 1,
 503         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 504         type => 'string',
 505         format => $agent_fmt,
 506     },
 507     kvm => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable KVM hardware virtualization.",
 511         default => 1,
 512     },
 513     tdf => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable time drift fix.",
 517         default => 0,
 518     },
 519     localtime => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 523             ." the `ostype` indicates a Microsoft Windows OS.",
 524     },
 525     freeze => {
 526         optional => 1,
 527         type => 'boolean',
 528         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 529     },
 530     vga => {
 531         optional => 1,
 532         type => 'string', format => $vga_fmt,
 533         description => "Configure the VGA hardware.",
 534         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 535             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 536             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 537             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 538             ." display server. For win* OS you can select how many independent displays you want,"
 539             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 540             ." using a serial device as terminal.",
 541     },
 542     watchdog => {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-watchdog',
 545         description => "Create a virtual hardware watchdog device.",
 546         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 547             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 548             ." else the watchdog will reset the guest (or execute the respective action specified)",
 549     },
 550     startdate => {
 551         optional => 1,
 552         type => 'string',
 553         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 554         description => "Set the initial date of the real time clock. Valid format for date are:"
 555             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 556         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 557         default => 'now',
 558     },
 559     startup =>  get_standard_option('pve-startup-order'),
 560     template => {
 561         optional => 1,
 562         type => 'boolean',
 563         description => "Enable/disable Template.",
 564         default => 0,
 565     },
 566     args => {
 567         optional => 1,
 568         type => 'string',
 569         description => "Arbitrary arguments passed to kvm.",
 570         verbose_description => <<EODESCR,
 571 Arbitrary arguments passed to kvm, for example:
 572
 573 args: -no-reboot -smbios 'type=0,vendor=FOO'
 574
 575 NOTE: this option is for experts only.
 576 EODESCR
 577     },
 578     tablet => {
 579         optional => 1,
 580         type => 'boolean',
 581         default => 1,
 582         description => "Enable/disable the USB tablet device.",
 583         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 584             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 585             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 586             ." may consider disabling this to save some context switches. This is turned off by"
 587             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 588     },
 589     migrate_speed => {
 590         optional => 1,
 591         type => 'integer',
 592         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 593         minimum => 0,
 594         default => 0,
 595     },
 596     migrate_downtime => {
 597         optional => 1,
 598         type => 'number',
 599         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 600         minimum => 0,
 601         default => 0.1,
 602     },
 603     cdrom => {
 604         optional => 1,
 605         type => 'string', format => 'pve-qm-ide',
 606         typetext => '<volume>',
 607         description => "This is an alias for option -ide2",
 608     },
 609     cpu => {
 610         optional => 1,
 611         description => "Emulated CPU type.",
 612         type => 'string',
 613         format => 'pve-vm-cpu-conf',
 614     },
 615     parent => get_standard_option('pve-snapshot-name', {
 616         optional => 1,
 617         description => "Parent snapshot name. This is used internally, and should not be modified.",
 618     }),
 619     snaptime => {
 620         optional => 1,
 621         description => "Timestamp for snapshots.",
 622         type => 'integer',
 623         minimum => 0,
 624     },
 625     vmstate => {
 626         optional => 1,
 627         type => 'string', format => 'pve-volume-id',
 628         description => "Reference to a volume which stores the VM state. This is used internally"
 629             ." for snapshots.",
 630     },
 631     vmstatestorage => get_standard_option('pve-storage-id', {
 632         description => "Default storage for VM state volumes/files.",
 633         optional => 1,
 634     }),
 635     runningmachine => get_standard_option('pve-qemu-machine', {
 636         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 637             ." for snapshots.",
 638     }),
 639     runningcpu => {
 640         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 641             ." internally for snapshots.",
 642         optional => 1,
 643         type => 'string',
 644         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 645         format_description => 'QEMU -cpu parameter'
 646     },
 647     machine => get_standard_option('pve-qemu-machine'),
 648     arch => {
 649         description => "Virtual processor architecture. Defaults to the host.",
 650         optional => 1,
 651         type => 'string',
 652         enum => [qw(x86_64 aarch64)],
 653     },
 654     smbios1 => {
 655         description => "Specify SMBIOS type 1 fields.",
 656         type => 'string', format => 'pve-qm-smbios1',
 657         maxLength => 512,
 658         optional => 1,
 659     },
 660     protection => {
 661         optional => 1,
 662         type => 'boolean',
 663         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 664             ." remove disk operations.",
 665         default => 0,
 666     },
 667     bios => {
 668         optional => 1,
 669         type => 'string',
 670         enum => [ qw(seabios ovmf) ],
 671         description => "Select BIOS implementation.",
 672         default => 'seabios',
 673     },
 674     vmgenid => {
 675         type => 'string',
 676         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 677         format_description => 'UUID',
 678         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 679             ." to disable explicitly.",
 680         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 681             ." value identifier to the guest OS. This allows to notify the guest operating system"
 682             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 683             ." execution or creation from a template). The guest operating system notices the"
 684             ." change, and is then able to react as appropriate by marking its copies of"
 685             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 686             ."Note that auto-creation only works when done through API/CLI create or update methods"
 687             .", but not when manually editing the config file.",
 688         default => "1 (autogenerated)",
 689         optional => 1,
 690     },
 691     hookscript => {
 692         type => 'string',
 693         format => 'pve-volume-id',
 694         optional => 1,
 695         description => "Script that will be executed during various steps in the vms lifetime.",
 696     },
 697     ivshmem => {
 698         type => 'string',
 699         format => $ivshmem_fmt,
 700         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 701             ." the host.",
 702         optional => 1,
 703     },
 704     audio0 => {
 705         type => 'string',
 706         format => $audio_fmt,
 707         description => "Configure a audio device, useful in combination with QXL/Spice.",
 708         optional => 1
 709     },
 710     spice_enhancements => {
 711         type => 'string',
 712         format => $spice_enhancements_fmt,
 713         description => "Configure additional enhancements for SPICE.",
 714         optional => 1
 715     },
 716     tags => {
 717         type => 'string', format => 'pve-tag-list',
 718         description => 'Tags of the VM. This is only meta information.',
 719         optional => 1,
 720     },
 721     rng0 => {
 722         type => 'string',
 723         format => $rng_fmt,
 724         description => "Configure a VirtIO-based Random Number Generator.",
 725         optional => 1,
 726     },
 727     meta => {
 728         type => 'string',
 729         format => $meta_info_fmt,
 730         description => "Some (read-only) meta-information about this guest.",
 731         optional => 1,
 732     },
 733     affinity => {
 734         type => 'string', format => 'pve-cpuset',
 735         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 736         optional => 1,
 737     },
 738 };
 739
 740 my $cicustom_fmt = {
 741     meta => {
 742         type => 'string',
 743         optional => 1,
 744         description => 'Specify a custom file containing all meta data passed to the VM via"
 745             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 746         format => 'pve-volume-id',
 747         format_description => 'volume',
 748     },
 749     network => {
 750         type => 'string',
 751         optional => 1,
 752         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 753         format => 'pve-volume-id',
 754         format_description => 'volume',
 755     },
 756     user => {
 757         type => 'string',
 758         optional => 1,
 759         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 760         format => 'pve-volume-id',
 761         format_description => 'volume',
 762     },
 763     vendor => {
 764         type => 'string',
 765         optional => 1,
 766         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 767         format => 'pve-volume-id',
 768         format_description => 'volume',
 769     },
 770 };
 771 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 772
 773 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 774 my $confdesc_cloudinit = {
 775     citype => {
 776         optional => 1,
 777         type => 'string',
 778         description => 'Specifies the cloud-init configuration format. The default depends on the'
 779             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 780             .' and `configdrive2` for windows.',
 781         enum => ['configdrive2', 'nocloud', 'opennebula'],
 782     },
 783     ciuser => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: User name to change ssh keys and password for instead of the"
 787             ." image's configured default user.",
 788     },
 789     cipassword => {
 790         optional => 1,
 791         type => 'string',
 792         description => 'cloud-init: Password to assign the user. Using this is generally not'
 793             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 794             .' support hashed passwords.',
 795     },
 796     ciupgrade => {
 797         optional => 1,
 798         type => 'boolean',
 799         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 800         default => 1,
 801     },
 802     cicustom => {
 803         optional => 1,
 804         type => 'string',
 805         description => 'cloud-init: Specify custom files to replace the automatically generated'
 806             .' ones at start.',
 807         format => 'pve-qm-cicustom',
 808     },
 809     searchdomain => {
 810         optional => 1,
 811         type => 'string',
 812         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 813             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 814             .' are set.',
 815     },
 816     nameserver => {
 817         optional => 1,
 818         type => 'string', format => 'address-list',
 819         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 820             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 821             .' are set.',
 822     },
 823     sshkeys => {
 824         optional => 1,
 825         type => 'string',
 826         format => 'urlencoded',
 827         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 828     },
 829 };
 830
 831 # what about other qemu settings ?
 832 #cpu => 'string',
 833 #machine => 'string',
 834 #fda => 'file',
 835 #fdb => 'file',
 836 #mtdblock => 'file',
 837 #sd => 'file',
 838 #pflash => 'file',
 839 #snapshot => 'bool',
 840 #bootp => 'file',
 841 ##tftp => 'dir',
 842 ##smb => 'dir',
 843 #kernel => 'file',
 844 #append => 'string',
 845 #initrd => 'file',
 846 ##soundhw => 'string',
 847
 848 while (my ($k, $v) = each %$confdesc) {
 849     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 850 }
 851
 852 my $MAX_NETS = 32;
 853 my $MAX_SERIAL_PORTS = 4;
 854 my $MAX_PARALLEL_PORTS = 3;
 855
 856 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 857     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 858 }
 859
 860 my $nic_model_list = [
 861     'e1000',
 862     'e1000-82540em',
 863     'e1000-82544gc',
 864     'e1000-82545em',
 865     'e1000e',
 866     'i82551',
 867     'i82557b',
 868     'i82559er',
 869     'ne2k_isa',
 870     'ne2k_pci',
 871     'pcnet',
 872     'rtl8139',
 873     'virtio',
 874     'vmxnet3',
 875 ];
 876 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 877
 878 my $net_fmt_bridge_descr = <<__EOD__;
 879 Bridge to attach the network device to. The Proxmox VE standard bridge
 880 is called 'vmbr0'.
 881
 882 If you do not specify a bridge, we create a kvm user (NATed) network
 883 device, which provides DHCP and DNS services. The following addresses
 884 are used:
 885
 886  10.0.2.2   Gateway
 887  10.0.2.3   DNS Server
 888  10.0.2.4   SMB Server
 889
 890 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 891 __EOD__
 892
 893 my $net_fmt = {
 894     macaddr  => get_standard_option('mac-addr', {
 895         description => "MAC address. That address must be unique withing your network. This is"
 896             ." automatically generated if not specified.",
 897     }),
 898     model => {
 899         type => 'string',
 900         description => "Network Card Model. The 'virtio' model provides the best performance with"
 901             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 902             ." best to use 'e1000'.",
 903         enum => $nic_model_list,
 904         default_key => 1,
 905     },
 906     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 907     bridge => get_standard_option('pve-bridge-id', {
 908         description => $net_fmt_bridge_descr,
 909         optional => 1,
 910     }),
 911     queues => {
 912         type => 'integer',
 913         minimum => 0, maximum => 64,
 914         description => 'Number of packet queues to be used on the device.',
 915         optional => 1,
 916     },
 917     rate => {
 918         type => 'number',
 919         minimum => 0,
 920         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 921         optional => 1,
 922     },
 923     tag => {
 924         type => 'integer',
 925         minimum => 1, maximum => 4094,
 926         description => 'VLAN tag to apply to packets on this interface.',
 927         optional => 1,
 928     },
 929     trunks => {
 930         type => 'string',
 931         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 932         description => 'VLAN trunks to pass through this interface.',
 933         format_description => 'vlanid[;vlanid...]',
 934         optional => 1,
 935     },
 936     firewall => {
 937         type => 'boolean',
 938         description => 'Whether this interface should be protected by the firewall.',
 939         optional => 1,
 940     },
 941     link_down => {
 942         type => 'boolean',
 943         description => 'Whether this interface should be disconnected (like pulling the plug).',
 944         optional => 1,
 945     },
 946     mtu => {
 947         type => 'integer',
 948         minimum => 1, maximum => 65520,
 949         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 950         optional => 1,
 951     },
 952 };
 953
 954 my $netdesc = {
 955     optional => 1,
 956     type => 'string', format => $net_fmt,
 957     description => "Specify network devices.",
 958 };
 959
 960 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 961
 962 my $ipconfig_fmt = {
 963     ip => {
 964         type => 'string',
 965         format => 'pve-ipv4-config',
 966         format_description => 'IPv4Format/CIDR',
 967         description => 'IPv4 address in CIDR format.',
 968         optional => 1,
 969         default => 'dhcp',
 970     },
 971     gw => {
 972         type => 'string',
 973         format => 'ipv4',
 974         format_description => 'GatewayIPv4',
 975         description => 'Default gateway for IPv4 traffic.',
 976         optional => 1,
 977         requires => 'ip',
 978     },
 979     ip6 => {
 980         type => 'string',
 981         format => 'pve-ipv6-config',
 982         format_description => 'IPv6Format/CIDR',
 983         description => 'IPv6 address in CIDR format.',
 984         optional => 1,
 985         default => 'dhcp',
 986     },
 987     gw6 => {
 988         type => 'string',
 989         format => 'ipv6',
 990         format_description => 'GatewayIPv6',
 991         description => 'Default gateway for IPv6 traffic.',
 992         optional => 1,
 993         requires => 'ip6',
 994     },
 995 };
 996 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 997 my $ipconfigdesc = {
 998     optional => 1,
 999     type => 'string', format => 'pve-qm-ipconfig',
1000     description => <<'EODESCR',
1001 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1002
1003 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1004
1005 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1006 gateway should be provided.
1007 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1008 cloud-init 19.4 or newer.
1009
1010 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1011 dhcp on IPv4.
1012 EODESCR
1013 };
1014 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1015
1016 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1017     $confdesc->{"net$i"} = $netdesc;
1018     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1019 }
1020
1021 foreach my $key (keys %$confdesc_cloudinit) {
1022     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1023 }
1024
1025 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1026 sub pve_verify_cpuset {
1027     my ($set_text, $noerr) = @_;
1028
1029     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1030
1031     if ($@) {
1032         return if $noerr;
1033         die "unable to parse cpuset option\n";
1034     }
1035
1036     return PVE::CpuSet->new($members)->short_string();
1037 }
1038
1039 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1040 sub verify_volume_id_or_qm_path {
1041     my ($volid, $noerr) = @_;
1042
1043     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1044
1045     return verify_volume_id_or_absolute_path($volid, $noerr);
1046 }
1047
1048 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1049 sub verify_volume_id_or_absolute_path {
1050     my ($volid, $noerr) = @_;
1051
1052     return $volid if $volid =~ m|^/|;
1053
1054     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1055     if ($@) {
1056         return if $noerr;
1057         die $@;
1058     }
1059     return $volid;
1060 }
1061
1062 my $serialdesc = {
1063         optional => 1,
1064         type => 'string',
1065         pattern => '(/dev/.+|socket)',
1066         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1067         verbose_description =>  <<EODESCR,
1068 Create a serial device inside the VM (n is 0 to 3), and pass through a
1069 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1070 host side (use 'qm terminal' to open a terminal connection).
1071
1072 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1073 use with special care.
1074
1075 CAUTION: Experimental! User reported problems with this option.
1076 EODESCR
1077 };
1078
1079 my $paralleldesc= {
1080         optional => 1,
1081         type => 'string',
1082         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1083         description =>  "Map host parallel devices (n is 0 to 2).",
1084         verbose_description =>  <<EODESCR,
1085 Map host parallel devices (n is 0 to 2).
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1095     $confdesc->{"parallel$i"} = $paralleldesc;
1096 }
1097
1098 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1099     $confdesc->{"serial$i"} = $serialdesc;
1100 }
1101
1102 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1103     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1104 }
1105
1106 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1107     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1108 }
1109
1110 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1111     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1112 }
1113
1114 my $boot_fmt = {
1115     legacy => {
1116         optional => 1,
1117         default_key => 1,
1118         type => 'string',
1119         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1120                      . " Deprecated, use 'order=' instead.",
1121         pattern => '[acdn]{1,4}',
1122         format_description => "[acdn]{1,4}",
1123
1124         # note: this is also the fallback if boot: is not given at all
1125         default => 'cdn',
1126     },
1127     order => {
1128         optional => 1,
1129         type => 'string',
1130         format => 'pve-qm-bootdev-list',
1131         format_description => "device[;device...]",
1132         description => <<EODESC,
1133 The guest will attempt to boot from devices in the order they appear here.
1134
1135 Disks, optical drives and passed-through storage USB devices will be directly
1136 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1137 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1138
1139 Note that only devices in this list will be marked as bootable and thus loaded
1140 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1141 (e.g. software-raid), you need to specify all of them here.
1142
1143 Overrides the deprecated 'legacy=[acdn]*' value when given.
1144 EODESC
1145     },
1146 };
1147 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1148
1149 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1150 sub verify_bootdev {
1151     my ($dev, $noerr) = @_;
1152
1153     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1154     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1155
1156     my $check = sub {
1157         my ($base) = @_;
1158         return 0 if $dev !~ m/^$base\d+$/;
1159         return 0 if !$confdesc->{$dev};
1160         return 1;
1161     };
1162
1163     return $dev if $check->("net");
1164     return $dev if $check->("usb");
1165     return $dev if $check->("hostpci");
1166
1167     return if $noerr;
1168     die "invalid boot device '$dev'\n";
1169 }
1170
1171 sub print_bootorder {
1172     my ($devs) = @_;
1173     return "" if !@$devs;
1174     my $data = { order => join(';', @$devs) };
1175     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1176 }
1177
1178 my $kvm_api_version = 0;
1179
1180 sub kvm_version {
1181     return $kvm_api_version if $kvm_api_version;
1182
1183     open my $fh, '<', '/dev/kvm' or return;
1184
1185     # 0xae00 => KVM_GET_API_VERSION
1186     $kvm_api_version = ioctl($fh, 0xae00, 0);
1187     close($fh);
1188
1189     return $kvm_api_version;
1190 }
1191
1192 my $kvm_user_version = {};
1193 my $kvm_mtime = {};
1194
1195 sub kvm_user_version {
1196     my ($binary) = @_;
1197
1198     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1199     my $st = stat($binary);
1200
1201     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1202     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1203         $cachedmtime == $st->mtime;
1204
1205     $kvm_user_version->{$binary} = 'unknown';
1206     $kvm_mtime->{$binary} = $st->mtime;
1207
1208     my $code = sub {
1209         my $line = shift;
1210         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1211             $kvm_user_version->{$binary} = $2;
1212         }
1213     };
1214
1215     eval { run_command([$binary, '--version'], outfunc => $code); };
1216     warn $@ if $@;
1217
1218     return $kvm_user_version->{$binary};
1219
1220 }
1221 my sub extract_version {
1222     my ($machine_type, $version) = @_;
1223     $version = kvm_user_version() if !defined($version);
1224     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1225 }
1226
1227 sub kernel_has_vhost_net {
1228     return -c '/dev/vhost-net';
1229 }
1230
1231 sub option_exists {
1232     my $key = shift;
1233     return defined($confdesc->{$key});
1234 }
1235
1236 my $cdrom_path;
1237 sub get_cdrom_path {
1238
1239     return $cdrom_path if defined($cdrom_path);
1240
1241     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1242
1243     if (!defined($cdrom_path)) {
1244         log_warn("no physical CD-ROM available, ignoring");
1245         $cdrom_path = '';
1246     }
1247
1248     return $cdrom_path;
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269      if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1314             if !$vtype;
1315         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1316         verify_media_type($opt, $vtype, $drive->{media});
1317         $drive->{file} = $volid;
1318     }
1319
1320     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1321 }
1322
1323 sub parse_hotplug_features {
1324     my ($data) = @_;
1325
1326     my $res = {};
1327
1328     return $res if $data eq '0';
1329
1330     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1331
1332     foreach my $feature (PVE::Tools::split_list($data)) {
1333         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1334             $res->{$1} = 1;
1335         } else {
1336             die "invalid hotplug feature '$feature'\n";
1337         }
1338     }
1339     return $res;
1340 }
1341
1342 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1343 sub pve_verify_hotplug_features {
1344     my ($value, $noerr) = @_;
1345
1346     return $value if parse_hotplug_features($value);
1347
1348     return if $noerr;
1349
1350     die "unable to parse hotplug option\n";
1351 }
1352
1353 sub assert_clipboard_config {
1354     my ($vga) = @_;
1355
1356     my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1357
1358     if (
1359         $vga->{'clipboard'}
1360         && $vga->{'clipboard'} eq 'vnc'
1361         && $vga->{type}
1362         && $vga->{type} !~ $clipboard_regex
1363     ) {
1364         die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1365     }
1366 }
1367
1368 sub print_tabletdevice_full {
1369     my ($conf, $arch) = @_;
1370
1371     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1372
1373     # we use uhci for old VMs because tablet driver was buggy in older qemu
1374     my $usbbus;
1375     if ($q35 || $arch eq 'aarch64') {
1376         $usbbus = 'ehci';
1377     } else {
1378         $usbbus = 'uhci';
1379     }
1380
1381     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1382 }
1383
1384 sub print_keyboarddevice_full {
1385     my ($conf, $arch) = @_;
1386
1387     return if $arch ne 'aarch64';
1388
1389     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1390 }
1391
1392 my sub get_drive_id {
1393     my ($drive) = @_;
1394     return "$drive->{interface}$drive->{index}";
1395 }
1396
1397 sub print_drivedevice_full {
1398     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1399
1400     my $device = '';
1401     my $maxdev = 0;
1402
1403     my $drive_id = get_drive_id($drive);
1404     if ($drive->{interface} eq 'virtio') {
1405         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1406         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1407         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1408     } elsif ($drive->{interface} eq 'scsi') {
1409
1410         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1411         my $unit = $drive->{index} % $maxdev;
1412
1413         my $machine_version = extract_version($machine_type, kvm_user_version());
1414         my $devicetype  = PVE::QemuServer::Drive::get_scsi_devicetype(
1415             $drive, $storecfg, $machine_version);
1416
1417         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1418             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1419         } else {
1420             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1421                 .",lun=$drive->{index}";
1422         }
1423         $device .= ",drive=drive-$drive_id,id=$drive_id";
1424
1425         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1426             $device .= ",rotation_rate=1";
1427         }
1428         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1429
1430     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1431         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1432         my $controller = int($drive->{index} / $maxdev);
1433         my $unit = $drive->{index} % $maxdev;
1434
1435         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1436         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1437         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1438         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1439         # were before. Move odd ones up by 2 where they don't clash.
1440         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1441             $controller += 2 * ($unit % 2);
1442             $unit = 0;
1443         }
1444
1445         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1446
1447         $device = "ide-$devicetype";
1448         if ($drive->{interface} eq 'ide') {
1449             $device .= ",bus=ide.$controller,unit=$unit";
1450         } else {
1451             $device .= ",bus=ahci$controller.$unit";
1452         }
1453         $device .= ",drive=drive-$drive_id,id=$drive_id";
1454
1455         if ($devicetype eq 'hd') {
1456             if (my $model = $drive->{model}) {
1457                 $model = URI::Escape::uri_unescape($model);
1458                 $device .= ",model=$model";
1459             }
1460             if ($drive->{ssd}) {
1461                 $device .= ",rotation_rate=1";
1462             }
1463         }
1464         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1465     } elsif ($drive->{interface} eq 'usb') {
1466         die "implement me";
1467         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1468     } else {
1469         die "unsupported interface type";
1470     }
1471
1472     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1473
1474     if (my $serial = $drive->{serial}) {
1475         $serial = URI::Escape::uri_unescape($serial);
1476         $device .= ",serial=$serial";
1477     }
1478
1479
1480     return $device;
1481 }
1482
1483 sub get_initiator_name {
1484     my $initiator;
1485
1486     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1487     while (defined(my $line = <$fh>)) {
1488         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1489         $initiator = $1;
1490         last;
1491     }
1492     $fh->close();
1493
1494     return $initiator;
1495 }
1496
1497 my sub storage_allows_io_uring_default {
1498     my ($scfg, $cache_direct) = @_;
1499
1500     # io_uring with cache mode writeback or writethrough on krbd will hang...
1501     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1502
1503     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1504     # sometimes, just plain disable...
1505     return if $scfg && $scfg->{type} eq 'lvm';
1506
1507     # io_uring causes problems when used with CIFS since kernel 5.15
1508     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1509     return if $scfg && $scfg->{type} eq 'cifs';
1510
1511     return 1;
1512 }
1513
1514 my sub drive_uses_cache_direct {
1515     my ($drive, $scfg) = @_;
1516
1517     my $cache_direct = 0;
1518
1519     if (my $cache = $drive->{cache}) {
1520         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1521     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1522         $cache_direct = 1;
1523     }
1524
1525     return $cache_direct;
1526 }
1527
1528 sub print_drive_commandline_full {
1529     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1530
1531     my $path;
1532     my $volid = $drive->{file};
1533     my $format = $drive->{format};
1534     my $drive_id = get_drive_id($drive);
1535
1536     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1537     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1538
1539     if (drive_is_cdrom($drive)) {
1540         $path = get_iso_path($storecfg, $vmid, $volid);
1541         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1542     } else {
1543         if ($storeid) {
1544             $path = PVE::Storage::path($storecfg, $volid);
1545             $format //= qemu_img_format($scfg, $volname);
1546         } else {
1547             $path = $volid;
1548             $format //= "raw";
1549         }
1550    }
1551
1552    my $is_rbd = $path =~ m/^rbd:/;
1553
1554     my $opts = '';
1555     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1556     foreach my $o (@qemu_drive_options) {
1557         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1558     }
1559
1560     # snapshot only accepts on|off
1561     if (defined($drive->{snapshot})) {
1562         my $v = $drive->{snapshot} ? 'on' : 'off';
1563         $opts .= ",snapshot=$v";
1564     }
1565
1566     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1567         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1568     }
1569
1570     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1571         my ($dir, $qmpname) = @$type;
1572         if (my $v = $drive->{"mbps$dir"}) {
1573             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1574         }
1575         if (my $v = $drive->{"mbps${dir}_max"}) {
1576             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1577         }
1578         if (my $v = $drive->{"bps${dir}_max_length"}) {
1579             $opts .= ",throttling.bps$qmpname-max-length=$v";
1580         }
1581         if (my $v = $drive->{"iops${dir}"}) {
1582             $opts .= ",throttling.iops$qmpname=$v";
1583         }
1584         if (my $v = $drive->{"iops${dir}_max"}) {
1585             $opts .= ",throttling.iops$qmpname-max=$v";
1586         }
1587         if (my $v = $drive->{"iops${dir}_max_length"}) {
1588             $opts .= ",throttling.iops$qmpname-max-length=$v";
1589         }
1590     }
1591
1592     if ($pbs_name) {
1593         $format = "rbd" if $is_rbd;
1594         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1595             if !$format;
1596         $opts .= ",format=alloc-track,file.driver=$format";
1597     } elsif ($format) {
1598         $opts .= ",format=$format";
1599     }
1600
1601     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1602
1603     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1604
1605     if (!$drive->{aio}) {
1606         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1607             # io_uring supports all cache modes
1608             $opts .= ",aio=io_uring";
1609         } else {
1610             # aio native works only with O_DIRECT
1611             if($cache_direct) {
1612                 $opts .= ",aio=native";
1613             } else {
1614                 $opts .= ",aio=threads";
1615             }
1616         }
1617     }
1618
1619     if (!drive_is_cdrom($drive)) {
1620         my $detectzeroes;
1621         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1622             $detectzeroes = 'off';
1623         } elsif ($drive->{discard}) {
1624             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1625         } else {
1626             # This used to be our default with discard not being specified:
1627             $detectzeroes = 'on';
1628         }
1629
1630         # note: 'detect-zeroes' works per blockdev and we want it to persist
1631         # after the alloc-track is removed, so put it on 'file' directly
1632         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1633         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1634     }
1635
1636     if ($pbs_name) {
1637         $opts .= ",backing=$pbs_name";
1638         $opts .= ",auto-remove=on";
1639     }
1640
1641     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1642     my $file_param = "file";
1643     if ($pbs_name) {
1644         # non-rbd drivers require the underlying file to be a seperate block
1645         # node, so add a second .file indirection
1646         $file_param .= ".file" if !$is_rbd;
1647         $file_param .= ".filename";
1648     }
1649     my $pathinfo = $path ? "$file_param=$path," : '';
1650
1651     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1652 }
1653
1654 sub print_pbs_blockdev {
1655     my ($pbs_conf, $pbs_name) = @_;
1656     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1657     $blockdev .= ",repository=$pbs_conf->{repository}";
1658     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1659     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1660     $blockdev .= ",archive=$pbs_conf->{archive}";
1661     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1662     return $blockdev;
1663 }
1664
1665 sub print_netdevice_full {
1666     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1667
1668     my $device = $net->{model};
1669     if ($net->{model} eq 'virtio') {
1670          $device = 'virtio-net-pci';
1671      };
1672
1673     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1674     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1675     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1676         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1677         # and out of each queue plus one config interrupt and control vector queue
1678         my $vectors = $net->{queues} * 2 + 2;
1679         $tmpstr .= ",vectors=$vectors,mq=on";
1680         if (min_version($machine_version, 7, 1)) {
1681             $tmpstr .= ",packed=on";
1682         }
1683     }
1684
1685     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1686         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1687     }
1688
1689     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1690
1691     if (my $mtu = $net->{mtu}) {
1692         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1693             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1694             if ($mtu == 1) {
1695                  $mtu = $bridge_mtu;
1696             } elsif ($mtu < 576) {
1697                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1698             } elsif ($mtu > $bridge_mtu) {
1699                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1700             }
1701             $tmpstr .= ",host_mtu=$mtu";
1702         } else {
1703             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1704         }
1705     }
1706
1707     if ($use_old_bios_files) {
1708         my $romfile;
1709         if ($device eq 'virtio-net-pci') {
1710             $romfile = 'pxe-virtio.rom';
1711         } elsif ($device eq 'e1000') {
1712             $romfile = 'pxe-e1000.rom';
1713         } elsif ($device eq 'e1000e') {
1714             $romfile = 'pxe-e1000e.rom';
1715         } elsif ($device eq 'ne2k') {
1716             $romfile = 'pxe-ne2k_pci.rom';
1717         } elsif ($device eq 'pcnet') {
1718             $romfile = 'pxe-pcnet.rom';
1719         } elsif ($device eq 'rtl8139') {
1720             $romfile = 'pxe-rtl8139.rom';
1721         }
1722         $tmpstr .= ",romfile=$romfile" if $romfile;
1723     }
1724
1725     return $tmpstr;
1726 }
1727
1728 sub print_netdev_full {
1729     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1730
1731     my $i = '';
1732     if ($netid =~ m/^net(\d+)$/) {
1733         $i = int($1);
1734     }
1735
1736     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1737
1738     my $ifname = "tap${vmid}i$i";
1739
1740     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1741     die "interface name '$ifname' is too long (max 15 character)\n"
1742         if length($ifname) >= 16;
1743
1744     my $vhostparam = '';
1745     if (is_native($arch)) {
1746         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1747     }
1748
1749     my $vmname = $conf->{name} || "vm$vmid";
1750
1751     my $netdev = "";
1752     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1753
1754     if ($net->{bridge}) {
1755         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1756             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1757     } else {
1758         $netdev = "type=user,id=$netid,hostname=$vmname";
1759     }
1760
1761     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1762
1763     return $netdev;
1764 }
1765
1766 my $vga_map = {
1767     'cirrus' => 'cirrus-vga',
1768     'std' => 'VGA',
1769     'vmware' => 'vmware-svga',
1770     'virtio' => 'virtio-vga',
1771     'virtio-gl' => 'virtio-vga-gl',
1772 };
1773
1774 sub print_vga_device {
1775     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1776
1777     my $type = $vga_map->{$vga->{type}};
1778     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1779         $type = 'virtio-gpu';
1780     }
1781     my $vgamem_mb = $vga->{memory};
1782
1783     my $max_outputs = '';
1784     if ($qxlnum) {
1785         $type = $id ? 'qxl' : 'qxl-vga';
1786
1787         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1788             # set max outputs so linux can have up to 4 qxl displays with one device
1789             if (min_version($machine_version, 4, 1)) {
1790                 $max_outputs = ",max_outputs=4";
1791             }
1792         }
1793     }
1794
1795     die "no devicetype for $vga->{type}\n" if !$type;
1796
1797     my $memory = "";
1798     if ($vgamem_mb) {
1799         if ($vga->{type} =~ /^virtio/) {
1800             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1801             $memory = ",max_hostmem=$bytes";
1802         } elsif ($qxlnum) {
1803             # from https://www.spice-space.org/multiple-monitors.html
1804             $memory = ",vgamem_mb=$vga->{memory}";
1805             my $ram = $vgamem_mb * 4;
1806             my $vram = $vgamem_mb * 2;
1807             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1808         } else {
1809             $memory = ",vgamem_mb=$vga->{memory}";
1810         }
1811     } elsif ($qxlnum && $id) {
1812         $memory = ",ram_size=67108864,vram_size=33554432";
1813     }
1814
1815     my $edidoff = "";
1816     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1817         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1818     }
1819
1820     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1821     my $vgaid = "vga" . ($id // '');
1822     my $pciaddr;
1823     if ($q35 && $vgaid eq 'vga') {
1824         # the first display uses pcie.0 bus on q35 machines
1825         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1826     } else {
1827         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1828     }
1829
1830     if ($vga->{type} eq 'virtio-gl') {
1831         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1832         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1833             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1834
1835         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1836             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1837     }
1838
1839     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1840 }
1841
1842 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1843 sub parse_net {
1844     my ($data, $disable_mac_autogen) = @_;
1845
1846     my $res = eval { parse_property_string($net_fmt, $data) };
1847     if ($@) {
1848         warn $@;
1849         return;
1850     }
1851     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1852         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1853         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1854     }
1855     return $res;
1856 }
1857
1858 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1859 sub parse_ipconfig {
1860     my ($data) = @_;
1861
1862     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1863     if ($@) {
1864         warn $@;
1865         return;
1866     }
1867
1868     if ($res->{gw} && !$res->{ip}) {
1869         warn 'gateway specified without specifying an IP address';
1870         return;
1871     }
1872     if ($res->{gw6} && !$res->{ip6}) {
1873         warn 'IPv6 gateway specified without specifying an IPv6 address';
1874         return;
1875     }
1876     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1877         warn 'gateway specified together with DHCP';
1878         return;
1879     }
1880     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1881         # gw6 + auto/dhcp
1882         warn "IPv6 gateway specified together with $res->{ip6} address";
1883         return;
1884     }
1885
1886     if (!$res->{ip} && !$res->{ip6}) {
1887         return { ip => 'dhcp', ip6 => 'dhcp' };
1888     }
1889
1890     return $res;
1891 }
1892
1893 sub print_net {
1894     my $net = shift;
1895
1896     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1897 }
1898
1899 sub add_random_macs {
1900     my ($settings) = @_;
1901
1902     foreach my $opt (keys %$settings) {
1903         next if $opt !~ m/^net(\d+)$/;
1904         my $net = parse_net($settings->{$opt});
1905         next if !$net;
1906         $settings->{$opt} = print_net($net);
1907     }
1908 }
1909
1910 sub vm_is_volid_owner {
1911     my ($storecfg, $vmid, $volid) = @_;
1912
1913     if ($volid !~  m|^/|) {
1914         my ($path, $owner);
1915         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1916         if ($owner && ($owner == $vmid)) {
1917             return 1;
1918         }
1919     }
1920
1921     return;
1922 }
1923
1924 sub vmconfig_register_unused_drive {
1925     my ($storecfg, $vmid, $conf, $drive) = @_;
1926
1927     if (drive_is_cloudinit($drive)) {
1928         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1929         warn $@ if $@;
1930         delete $conf->{cloudinit};
1931     } elsif (!drive_is_cdrom($drive)) {
1932         my $volid = $drive->{file};
1933         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1934             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1935         }
1936     }
1937 }
1938
1939 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1940 my $smbios1_fmt = {
1941     uuid => {
1942         type => 'string',
1943         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1944         format_description => 'UUID',
1945         description => "Set SMBIOS1 UUID.",
1946         optional => 1,
1947     },
1948     version => {
1949         type => 'string',
1950         pattern => '[A-Za-z0-9+\/]+={0,2}',
1951         format_description => 'Base64 encoded string',
1952         description => "Set SMBIOS1 version.",
1953         optional => 1,
1954     },
1955     serial => {
1956         type => 'string',
1957         pattern => '[A-Za-z0-9+\/]+={0,2}',
1958         format_description => 'Base64 encoded string',
1959         description => "Set SMBIOS1 serial number.",
1960         optional => 1,
1961     },
1962     manufacturer => {
1963         type => 'string',
1964         pattern => '[A-Za-z0-9+\/]+={0,2}',
1965         format_description => 'Base64 encoded string',
1966         description => "Set SMBIOS1 manufacturer.",
1967         optional => 1,
1968     },
1969     product => {
1970         type => 'string',
1971         pattern => '[A-Za-z0-9+\/]+={0,2}',
1972         format_description => 'Base64 encoded string',
1973         description => "Set SMBIOS1 product ID.",
1974         optional => 1,
1975     },
1976     sku => {
1977         type => 'string',
1978         pattern => '[A-Za-z0-9+\/]+={0,2}',
1979         format_description => 'Base64 encoded string',
1980         description => "Set SMBIOS1 SKU string.",
1981         optional => 1,
1982     },
1983     family => {
1984         type => 'string',
1985         pattern => '[A-Za-z0-9+\/]+={0,2}',
1986         format_description => 'Base64 encoded string',
1987         description => "Set SMBIOS1 family string.",
1988         optional => 1,
1989     },
1990     base64 => {
1991         type => 'boolean',
1992         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1993         optional => 1,
1994     },
1995 };
1996
1997 sub parse_smbios1 {
1998     my ($data) = @_;
1999
2000     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2001     warn $@ if $@;
2002     return $res;
2003 }
2004
2005 sub print_smbios1 {
2006     my ($smbios1) = @_;
2007     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2008 }
2009
2010 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2011
2012 sub parse_watchdog {
2013     my ($value) = @_;
2014
2015     return if !$value;
2016
2017     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2018     warn $@ if $@;
2019     return $res;
2020 }
2021
2022 sub parse_guest_agent {
2023     my ($conf) = @_;
2024
2025     return {} if !defined($conf->{agent});
2026
2027     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2028     warn $@ if $@;
2029
2030     # if the agent is disabled ignore the other potentially set properties
2031     return {} if !$res->{enabled};
2032     return $res;
2033 }
2034
2035 sub get_qga_key {
2036     my ($conf, $key) = @_;
2037     return undef if !defined($conf->{agent});
2038
2039     my $agent = parse_guest_agent($conf);
2040     return $agent->{$key};
2041 }
2042
2043 sub parse_vga {
2044     my ($value) = @_;
2045
2046     return {} if !$value;
2047     my $res = eval { parse_property_string($vga_fmt, $value) };
2048     warn $@ if $@;
2049     return $res;
2050 }
2051
2052 sub parse_rng {
2053     my ($value) = @_;
2054
2055     return if !$value;
2056
2057     my $res = eval { parse_property_string($rng_fmt, $value) };
2058     warn $@ if $@;
2059     return $res;
2060 }
2061
2062 sub parse_meta_info {
2063     my ($value) = @_;
2064
2065     return if !$value;
2066
2067     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2068     warn $@ if $@;
2069     return $res;
2070 }
2071
2072 sub new_meta_info_string {
2073     my () = @_; # for now do not allow to override any value
2074
2075     return PVE::JSONSchema::print_property_string(
2076         {
2077             'creation-qemu' => kvm_user_version(),
2078             ctime => "". int(time()),
2079         },
2080         $meta_info_fmt
2081     );
2082 }
2083
2084 sub qemu_created_version_fixups {
2085     my ($conf, $forcemachine, $kvmver) = @_;
2086
2087     my $meta = parse_meta_info($conf->{meta}) // {};
2088     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2089
2090     # check if we need to apply some handling for VMs that always use the latest machine version but
2091     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2092     # would be required (we do not want to pin machine version for non-windows OS type)
2093     if (
2094         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2095         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2096         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2097         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2098     ) {
2099         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2100         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2101             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2102             # and thus with the predictable interface naming of systemd
2103             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2104         }
2105     }
2106     return;
2107 }
2108
2109 # add JSON properties for create and set function
2110 sub json_config_properties {
2111     my ($prop, $with_disk_alloc) = @_;
2112
2113     my $skip_json_config_opts = {
2114         parent => 1,
2115         snaptime => 1,
2116         vmstate => 1,
2117         runningmachine => 1,
2118         runningcpu => 1,
2119         meta => 1,
2120     };
2121
2122     foreach my $opt (keys %$confdesc) {
2123         next if $skip_json_config_opts->{$opt};
2124
2125         if ($with_disk_alloc && is_valid_drivename($opt)) {
2126             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2127         } else {
2128             $prop->{$opt} = $confdesc->{$opt};
2129         }
2130     }
2131
2132     return $prop;
2133 }
2134
2135 # Properties that we can read from an OVF file
2136 sub json_ovf_properties {
2137     my $prop = {};
2138
2139     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2140         $prop->{$device} = {
2141             type => 'string',
2142             format => 'pve-volume-id-or-absolute-path',
2143             description => "Disk image that gets imported to $device",
2144             optional => 1,
2145         };
2146     }
2147
2148     $prop->{cores} = {
2149         type => 'integer',
2150         description => "The number of CPU cores.",
2151         optional => 1,
2152     };
2153     $prop->{memory} = {
2154         type => 'integer',
2155         description => "Amount of RAM for the VM in MB.",
2156         optional => 1,
2157     };
2158     $prop->{name} = {
2159         type => 'string',
2160         description => "Name of the VM.",
2161         optional => 1,
2162     };
2163
2164     return $prop;
2165 }
2166
2167 # return copy of $confdesc_cloudinit to generate documentation
2168 sub cloudinit_config_properties {
2169
2170     return dclone($confdesc_cloudinit);
2171 }
2172
2173 sub cloudinit_pending_properties {
2174     my $p = {
2175         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2176         name => 1,
2177     };
2178     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2179     return $p;
2180 }
2181
2182 sub check_type {
2183     my ($key, $value) = @_;
2184
2185     die "unknown setting '$key'\n" if !$confdesc->{$key};
2186
2187     my $type = $confdesc->{$key}->{type};
2188
2189     if (!defined($value)) {
2190         die "got undefined value\n";
2191     }
2192
2193     if ($value =~ m/[\n\r]/) {
2194         die "property contains a line feed\n";
2195     }
2196
2197     if ($type eq 'boolean') {
2198         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2199         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2200         die "type check ('boolean') failed - got '$value'\n";
2201     } elsif ($type eq 'integer') {
2202         return int($1) if $value =~ m/^(\d+)$/;
2203         die "type check ('integer') failed - got '$value'\n";
2204     } elsif ($type eq 'number') {
2205         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2206         die "type check ('number') failed - got '$value'\n";
2207     } elsif ($type eq 'string') {
2208         if (my $fmt = $confdesc->{$key}->{format}) {
2209             PVE::JSONSchema::check_format($fmt, $value);
2210             return $value;
2211         }
2212         $value =~ s/^\"(.*)\"$/$1/;
2213         return $value;
2214     } else {
2215         die "internal error"
2216     }
2217 }
2218
2219 sub destroy_vm {
2220     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2221
2222     my $conf = PVE::QemuConfig->load_config($vmid);
2223
2224     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2225         PVE::QemuConfig->check_lock($conf);
2226     }
2227
2228     if ($conf->{template}) {
2229         # check if any base image is still used by a linked clone
2230         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2231                 my ($ds, $drive) = @_;
2232                 return if drive_is_cdrom($drive);
2233
2234                 my $volid = $drive->{file};
2235                 return if !$volid || $volid =~ m|^/|;
2236
2237                 die "base volume '$volid' is still in use by linked cloned\n"
2238                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2239
2240         });
2241     }
2242
2243     my $volids = {};
2244     my $remove_owned_drive = sub {
2245         my ($ds, $drive) = @_;
2246         return if drive_is_cdrom($drive, 1);
2247
2248         my $volid = $drive->{file};
2249         return if !$volid || $volid =~ m|^/|;
2250         return if $volids->{$volid};
2251
2252         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2253         return if !$path || !$owner || ($owner != $vmid);
2254
2255         $volids->{$volid} = 1;
2256         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2257         warn "Could not remove disk '$volid', check manually: $@" if $@;
2258     };
2259
2260     # only remove disks owned by this VM (referenced in the config)
2261     my $include_opts = {
2262         include_unused => 1,
2263         extra_keys => ['vmstate'],
2264     };
2265     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2266
2267     for my $snap (values %{$conf->{snapshots}}) {
2268         next if !defined($snap->{vmstate});
2269         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2270         next if !defined($drive);
2271         $remove_owned_drive->('vmstate', $drive);
2272     }
2273
2274     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2275
2276     if ($purge_unreferenced) { # also remove unreferenced disk
2277         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2278         PVE::Storage::foreach_volid($vmdisks, sub {
2279             my ($volid, $sid, $volname, $d) = @_;
2280             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2281             warn $@ if $@;
2282         });
2283     }
2284
2285     eval { delete_ifaces_ipams_ips($conf, $vmid)};
2286     warn $@ if $@;
2287
2288     if (defined $replacement_conf) {
2289         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2290     } else {
2291         PVE::QemuConfig->destroy_config($vmid);
2292     }
2293 }
2294
2295 sub parse_vm_config {
2296     my ($filename, $raw, $strict) = @_;
2297
2298     return if !defined($raw);
2299
2300     my $res = {
2301         digest => Digest::SHA::sha1_hex($raw),
2302         snapshots => {},
2303         pending => {},
2304         cloudinit => {},
2305     };
2306
2307     my $handle_error = sub {
2308         my ($msg) = @_;
2309
2310         if ($strict) {
2311             die $msg;
2312         } else {
2313             warn $msg;
2314         }
2315     };
2316
2317     $filename =~ m|/qemu-server/(\d+)\.conf$|
2318         || die "got strange filename '$filename'";
2319
2320     my $vmid = $1;
2321
2322     my $conf = $res;
2323     my $descr;
2324     my $finish_description = sub {
2325         if (defined($descr)) {
2326             $descr =~ s/\s+$//;
2327             $conf->{description} = $descr;
2328         }
2329         $descr = undef;
2330     };
2331     my $section = '';
2332
2333     my @lines = split(/\n/, $raw);
2334     foreach my $line (@lines) {
2335         next if $line =~ m/^\s*$/;
2336
2337         if ($line =~ m/^\[PENDING\]\s*$/i) {
2338             $section = 'pending';
2339             $finish_description->();
2340             $conf = $res->{$section} = {};
2341             next;
2342         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2343             $section = 'cloudinit';
2344             $finish_description->();
2345             $conf = $res->{$section} = {};
2346             next;
2347
2348         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2349             $section = $1;
2350             $finish_description->();
2351             $conf = $res->{snapshots}->{$section} = {};
2352             next;
2353         }
2354
2355         if ($line =~ m/^\#(.*)$/) {
2356             $descr = '' if !defined($descr);
2357             $descr .= PVE::Tools::decode_text($1) . "\n";
2358             next;
2359         }
2360
2361         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2362             $descr = '' if !defined($descr);
2363             $descr .= PVE::Tools::decode_text($2);
2364         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2365             $conf->{snapstate} = $1;
2366         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2367             my $key = $1;
2368             my $value = $2;
2369             $conf->{$key} = $value;
2370         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2371             my $value = $1;
2372             if ($section eq 'pending') {
2373                 $conf->{delete} = $value; # we parse this later
2374             } else {
2375                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2376             }
2377         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2378             my $key = $1;
2379             my $value = $2;
2380             if ($section eq 'cloudinit') {
2381                 # ignore validation only used for informative purpose
2382                 $conf->{$key} = $value;
2383                 next;
2384             }
2385             eval { $value = check_type($key, $value); };
2386             if ($@) {
2387                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2388             } else {
2389                 $key = 'ide2' if $key eq 'cdrom';
2390                 my $fmt = $confdesc->{$key}->{format};
2391                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2392                     my $v = parse_drive($key, $value);
2393                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2394                         $v->{file} = $volid;
2395                         $value = print_drive($v);
2396                     } else {
2397                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2398                         next;
2399                     }
2400                 }
2401
2402                 $conf->{$key} = $value;
2403             }
2404         } else {
2405             $handle_error->("vm $vmid - unable to parse config: $line\n");
2406         }
2407     }
2408
2409     $finish_description->();
2410     delete $res->{snapstate}; # just to be sure
2411
2412     return $res;
2413 }
2414
2415 sub write_vm_config {
2416     my ($filename, $conf) = @_;
2417
2418     delete $conf->{snapstate}; # just to be sure
2419
2420     if ($conf->{cdrom}) {
2421         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2422         $conf->{ide2} = $conf->{cdrom};
2423         delete $conf->{cdrom};
2424     }
2425
2426     # we do not use 'smp' any longer
2427     if ($conf->{sockets}) {
2428         delete $conf->{smp};
2429     } elsif ($conf->{smp}) {
2430         $conf->{sockets} = $conf->{smp};
2431         delete $conf->{cores};
2432         delete $conf->{smp};
2433     }
2434
2435     my $used_volids = {};
2436
2437     my $cleanup_config = sub {
2438         my ($cref, $pending, $snapname) = @_;
2439
2440         foreach my $key (keys %$cref) {
2441             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2442                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2443             my $value = $cref->{$key};
2444             if ($key eq 'delete') {
2445                 die "propertry 'delete' is only allowed in [PENDING]\n"
2446                     if !$pending;
2447                 # fixme: check syntax?
2448                 next;
2449             }
2450             eval { $value = check_type($key, $value); };
2451             die "unable to parse value of '$key' - $@" if $@;
2452
2453             $cref->{$key} = $value;
2454
2455             if (!$snapname && is_valid_drivename($key)) {
2456                 my $drive = parse_drive($key, $value);
2457                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2458             }
2459         }
2460     };
2461
2462     &$cleanup_config($conf);
2463
2464     &$cleanup_config($conf->{pending}, 1);
2465
2466     foreach my $snapname (keys %{$conf->{snapshots}}) {
2467         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2468         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2469     }
2470
2471     # remove 'unusedX' settings if we re-add a volume
2472     foreach my $key (keys %$conf) {
2473         my $value = $conf->{$key};
2474         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2475             delete $conf->{$key};
2476         }
2477     }
2478
2479     my $generate_raw_config = sub {
2480         my ($conf, $pending) = @_;
2481
2482         my $raw = '';
2483
2484         # add description as comment to top of file
2485         if (defined(my $descr = $conf->{description})) {
2486             if ($descr) {
2487                 foreach my $cl (split(/\n/, $descr)) {
2488                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2489                 }
2490             } else {
2491                 $raw .= "#\n" if $pending;
2492             }
2493         }
2494
2495         foreach my $key (sort keys %$conf) {
2496             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2497             $raw .= "$key: $conf->{$key}\n";
2498         }
2499         return $raw;
2500     };
2501
2502     my $raw = &$generate_raw_config($conf);
2503
2504     if (scalar(keys %{$conf->{pending}})){
2505         $raw .= "\n[PENDING]\n";
2506         $raw .= &$generate_raw_config($conf->{pending}, 1);
2507     }
2508
2509     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2510         $raw .= "\n[special:cloudinit]\n";
2511         $raw .= &$generate_raw_config($conf->{cloudinit});
2512     }
2513
2514     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2515         $raw .= "\n[$snapname]\n";
2516         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2517     }
2518
2519     return $raw;
2520 }
2521
2522 sub load_defaults {
2523
2524     my $res = {};
2525
2526     # we use static defaults from our JSON schema configuration
2527     foreach my $key (keys %$confdesc) {
2528         if (defined(my $default = $confdesc->{$key}->{default})) {
2529             $res->{$key} = $default;
2530         }
2531     }
2532
2533     return $res;
2534 }
2535
2536 sub config_list {
2537     my $vmlist = PVE::Cluster::get_vmlist();
2538     my $res = {};
2539     return $res if !$vmlist || !$vmlist->{ids};
2540     my $ids = $vmlist->{ids};
2541     my $nodename = nodename();
2542
2543     foreach my $vmid (keys %$ids) {
2544         my $d = $ids->{$vmid};
2545         next if !$d->{node} || $d->{node} ne $nodename;
2546         next if !$d->{type} || $d->{type} ne 'qemu';
2547         $res->{$vmid}->{exists} = 1;
2548     }
2549     return $res;
2550 }
2551
2552 # test if VM uses local resources (to prevent migration)
2553 sub check_local_resources {
2554     my ($conf, $noerr) = @_;
2555
2556     my @loc_res = ();
2557     my $mapped_res = [];
2558
2559     my $nodelist = PVE::Cluster::get_nodelist();
2560     my $pci_map = PVE::Mapping::PCI::config();
2561     my $usb_map = PVE::Mapping::USB::config();
2562
2563     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2564
2565     my $add_missing_mapping = sub {
2566         my ($type, $key, $id) = @_;
2567         for my $node (@$nodelist) {
2568             my $entry;
2569             if ($type eq 'pci') {
2570                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2571             } elsif ($type eq 'usb') {
2572                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2573             }
2574             if (!scalar($entry->@*)) {
2575                 push @{$missing_mappings_by_node->{$node}}, $key;
2576             }
2577         }
2578     };
2579
2580     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2581     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2582
2583     push @loc_res, "ivshmem" if $conf->{ivshmem};
2584
2585     foreach my $k (keys %$conf) {
2586         if ($k =~ m/^usb/) {
2587             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2588             next if $entry->{host} =~ m/^spice$/i;
2589             if ($entry->{mapping}) {
2590                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2591                 push @$mapped_res, $k;
2592             }
2593         }
2594         if ($k =~ m/^hostpci/) {
2595             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2596             if ($entry->{mapping}) {
2597                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2598                 push @$mapped_res, $k;
2599             }
2600         }
2601         # sockets are safe: they will recreated be on the target side post-migrate
2602         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2603         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2604     }
2605
2606     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2607
2608     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2609 }
2610
2611 # check if used storages are available on all nodes (use by migrate)
2612 sub check_storage_availability {
2613     my ($storecfg, $conf, $node) = @_;
2614
2615     PVE::QemuConfig->foreach_volume($conf, sub {
2616         my ($ds, $drive) = @_;
2617
2618         my $volid = $drive->{file};
2619         return if !$volid;
2620
2621         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2622         return if !$sid;
2623
2624         # check if storage is available on both nodes
2625         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2626         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2627
2628         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2629
2630         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2631             if !$scfg->{content}->{$vtype};
2632    });
2633 }
2634
2635 # list nodes where all VM images are available (used by has_feature API)
2636 sub shared_nodes {
2637     my ($conf, $storecfg) = @_;
2638
2639     my $nodelist = PVE::Cluster::get_nodelist();
2640     my $nodehash = { map { $_ => 1 } @$nodelist };
2641     my $nodename = nodename();
2642
2643     PVE::QemuConfig->foreach_volume($conf, sub {
2644         my ($ds, $drive) = @_;
2645
2646         my $volid = $drive->{file};
2647         return if !$volid;
2648
2649         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2650         if ($storeid) {
2651             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2652             if ($scfg->{disable}) {
2653                 $nodehash = {};
2654             } elsif (my $avail = $scfg->{nodes}) {
2655                 foreach my $node (keys %$nodehash) {
2656                     delete $nodehash->{$node} if !$avail->{$node};
2657                 }
2658             } elsif (!$scfg->{shared}) {
2659                 foreach my $node (keys %$nodehash) {
2660                     delete $nodehash->{$node} if $node ne $nodename
2661                 }
2662             }
2663         }
2664     });
2665
2666     return $nodehash
2667 }
2668
2669 sub check_local_storage_availability {
2670     my ($conf, $storecfg) = @_;
2671
2672     my $nodelist = PVE::Cluster::get_nodelist();
2673     my $nodehash = { map { $_ => {} } @$nodelist };
2674
2675     PVE::QemuConfig->foreach_volume($conf, sub {
2676         my ($ds, $drive) = @_;
2677
2678         my $volid = $drive->{file};
2679         return if !$volid;
2680
2681         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2682         if ($storeid) {
2683             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2684
2685             if ($scfg->{disable}) {
2686                 foreach my $node (keys %$nodehash) {
2687                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2688                 }
2689             } elsif (my $avail = $scfg->{nodes}) {
2690                 foreach my $node (keys %$nodehash) {
2691                     if (!$avail->{$node}) {
2692                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2693                     }
2694                 }
2695             }
2696         }
2697     });
2698
2699     foreach my $node (values %$nodehash) {
2700         if (my $unavail = $node->{unavailable_storages}) {
2701             $node->{unavailable_storages} = [ sort keys %$unavail ];
2702         }
2703     }
2704
2705     return $nodehash
2706 }
2707
2708 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2709 sub check_running {
2710     my ($vmid, $nocheck, $node) = @_;
2711
2712     # $nocheck is set when called during a migration, in which case the config
2713     # file might still or already reside on the *other* node
2714     # - because rename has already happened, and current node is source
2715     # - because rename hasn't happened yet, and current node is target
2716     # - because rename has happened, current node is target, but hasn't yet
2717     # processed it yet
2718     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2719     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2720 }
2721
2722 sub vzlist {
2723
2724     my $vzlist = config_list();
2725
2726     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2727
2728     while (defined(my $de = $fd->read)) {
2729         next if $de !~ m/^(\d+)\.pid$/;
2730         my $vmid = $1;
2731         next if !defined($vzlist->{$vmid});
2732         if (my $pid = check_running($vmid)) {
2733             $vzlist->{$vmid}->{pid} = $pid;
2734         }
2735     }
2736
2737     return $vzlist;
2738 }
2739
2740 our $vmstatus_return_properties = {
2741     vmid => get_standard_option('pve-vmid'),
2742     status => {
2743         description => "QEMU process status.",
2744         type => 'string',
2745         enum => ['stopped', 'running'],
2746     },
2747     maxmem => {
2748         description => "Maximum memory in bytes.",
2749         type => 'integer',
2750         optional => 1,
2751         renderer => 'bytes',
2752     },
2753     maxdisk => {
2754         description => "Root disk size in bytes.",
2755         type => 'integer',
2756         optional => 1,
2757         renderer => 'bytes',
2758     },
2759     name => {
2760         description => "VM name.",
2761         type => 'string',
2762         optional => 1,
2763     },
2764     qmpstatus => {
2765         description => "VM run state from the 'query-status' QMP monitor command.",
2766         type => 'string',
2767         optional => 1,
2768     },
2769     pid => {
2770         description => "PID of running qemu process.",
2771         type => 'integer',
2772         optional => 1,
2773     },
2774     uptime => {
2775         description => "Uptime.",
2776         type => 'integer',
2777         optional => 1,
2778         renderer => 'duration',
2779     },
2780     cpus => {
2781         description => "Maximum usable CPUs.",
2782         type => 'number',
2783         optional => 1,
2784     },
2785     lock => {
2786         description => "The current config lock, if any.",
2787         type => 'string',
2788         optional => 1,
2789     },
2790     tags => {
2791         description  => "The current configured tags, if any",
2792         type => 'string',
2793         optional => 1,
2794     },
2795     'running-machine' => {
2796         description => "The currently running machine type (if running).",
2797         type => 'string',
2798         optional => 1,
2799     },
2800     'running-qemu' => {
2801         description => "The currently running QEMU version (if running).",
2802         type => 'string',
2803         optional => 1,
2804     },
2805 };
2806
2807 my $last_proc_pid_stat;
2808
2809 # get VM status information
2810 # This must be fast and should not block ($full == false)
2811 # We only query KVM using QMP if $full == true (this can be slow)
2812 sub vmstatus {
2813     my ($opt_vmid, $full) = @_;
2814
2815     my $res = {};
2816
2817     my $storecfg = PVE::Storage::config();
2818
2819     my $list = vzlist();
2820     my $defaults = load_defaults();
2821
2822     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2823
2824     my $cpucount = $cpuinfo->{cpus} || 1;
2825
2826     foreach my $vmid (keys %$list) {
2827         next if $opt_vmid && ($vmid ne $opt_vmid);
2828
2829         my $conf = PVE::QemuConfig->load_config($vmid);
2830
2831         my $d = { vmid => int($vmid) };
2832         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2833
2834         # fixme: better status?
2835         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2836
2837         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2838         if (defined($size)) {
2839             $d->{disk} = 0; # no info available
2840             $d->{maxdisk} = $size;
2841         } else {
2842             $d->{disk} = 0;
2843             $d->{maxdisk} = 0;
2844         }
2845
2846         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2847             * ($conf->{cores} || $defaults->{cores});
2848         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2849         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2850
2851         $d->{name} = $conf->{name} || "VM $vmid";
2852         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2853
2854         if ($conf->{balloon}) {
2855             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2856             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2857                 : $defaults->{shares};
2858         }
2859
2860         $d->{uptime} = 0;
2861         $d->{cpu} = 0;
2862         $d->{mem} = 0;
2863
2864         $d->{netout} = 0;
2865         $d->{netin} = 0;
2866
2867         $d->{diskread} = 0;
2868         $d->{diskwrite} = 0;
2869
2870         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2871
2872         $d->{serial} = 1 if conf_has_serial($conf);
2873         $d->{lock} = $conf->{lock} if $conf->{lock};
2874         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2875
2876         $res->{$vmid} = $d;
2877     }
2878
2879     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2880     foreach my $dev (keys %$netdev) {
2881         next if $dev !~ m/^tap([1-9]\d*)i/;
2882         my $vmid = $1;
2883         my $d = $res->{$vmid};
2884         next if !$d;
2885
2886         $d->{netout} += $netdev->{$dev}->{receive};
2887         $d->{netin} += $netdev->{$dev}->{transmit};
2888
2889         if ($full) {
2890             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2891             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2892         }
2893
2894     }
2895
2896     my $ctime = gettimeofday;
2897
2898     foreach my $vmid (keys %$list) {
2899
2900         my $d = $res->{$vmid};
2901         my $pid = $d->{pid};
2902         next if !$pid;
2903
2904         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2905         next if !$pstat; # not running
2906
2907         my $used = $pstat->{utime} + $pstat->{stime};
2908
2909         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2910
2911         if ($pstat->{vsize}) {
2912             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2913         }
2914
2915         my $old = $last_proc_pid_stat->{$pid};
2916         if (!$old) {
2917             $last_proc_pid_stat->{$pid} = {
2918                 time => $ctime,
2919                 used => $used,
2920                 cpu => 0,
2921             };
2922             next;
2923         }
2924
2925         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2926
2927         if ($dtime > 1000) {
2928             my $dutime = $used -  $old->{used};
2929
2930             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2931             $last_proc_pid_stat->{$pid} = {
2932                 time => $ctime,
2933                 used => $used,
2934                 cpu => $d->{cpu},
2935             };
2936         } else {
2937             $d->{cpu} = $old->{cpu};
2938         }
2939     }
2940
2941     return $res if !$full;
2942
2943     my $qmpclient = PVE::QMPClient->new();
2944
2945     my $ballooncb = sub {
2946         my ($vmid, $resp) = @_;
2947
2948         my $info = $resp->{'return'};
2949         return if !$info->{max_mem};
2950
2951         my $d = $res->{$vmid};
2952
2953         # use memory assigned to VM
2954         $d->{maxmem} = $info->{max_mem};
2955         $d->{balloon} = $info->{actual};
2956
2957         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2958             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2959             $d->{freemem} = $info->{free_mem};
2960         }
2961
2962         $d->{ballooninfo} = $info;
2963     };
2964
2965     my $blockstatscb = sub {
2966         my ($vmid, $resp) = @_;
2967         my $data = $resp->{'return'} || [];
2968         my $totalrdbytes = 0;
2969         my $totalwrbytes = 0;
2970
2971         for my $blockstat (@$data) {
2972             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2973             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2974
2975             $blockstat->{device} =~ s/drive-//;
2976             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2977         }
2978         $res->{$vmid}->{diskread} = $totalrdbytes;
2979         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2980     };
2981
2982     my $machinecb = sub {
2983         my ($vmid, $resp) = @_;
2984         my $data = $resp->{'return'} || [];
2985
2986         $res->{$vmid}->{'running-machine'} =
2987             PVE::QemuServer::Machine::current_from_query_machines($data);
2988     };
2989
2990     my $versioncb = sub {
2991         my ($vmid, $resp) = @_;
2992         my $data = $resp->{'return'} // {};
2993         my $version = 'unknown';
2994
2995         if (my $v = $data->{qemu}) {
2996             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2997         }
2998
2999         $res->{$vmid}->{'running-qemu'} = $version;
3000     };
3001
3002     my $statuscb = sub {
3003         my ($vmid, $resp) = @_;
3004
3005         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3006         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3007         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3008         # this fails if ballon driver is not loaded, so this must be
3009         # the last commnand (following command are aborted if this fails).
3010         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3011
3012         my $status = 'unknown';
3013         if (!defined($status = $resp->{'return'}->{status})) {
3014             warn "unable to get VM status\n";
3015             return;
3016         }
3017
3018         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3019     };
3020
3021     foreach my $vmid (keys %$list) {
3022         next if $opt_vmid && ($vmid ne $opt_vmid);
3023         next if !$res->{$vmid}->{pid}; # not running
3024         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3025     }
3026
3027     $qmpclient->queue_execute(undef, 2);
3028
3029     foreach my $vmid (keys %$list) {
3030         next if $opt_vmid && ($vmid ne $opt_vmid);
3031         next if !$res->{$vmid}->{pid}; #not running
3032
3033         # we can't use the $qmpclient since it might have already aborted on
3034         # 'query-balloon', but this might also fail for older versions...
3035         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3036         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3037     }
3038
3039     foreach my $vmid (keys %$list) {
3040         next if $opt_vmid && ($vmid ne $opt_vmid);
3041         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3042     }
3043
3044     return $res;
3045 }
3046
3047 sub conf_has_serial {
3048     my ($conf) = @_;
3049
3050     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3051         if ($conf->{"serial$i"}) {
3052             return 1;
3053         }
3054     }
3055
3056     return 0;
3057 }
3058
3059 sub conf_has_audio {
3060     my ($conf, $id) = @_;
3061
3062     $id //= 0;
3063     my $audio = $conf->{"audio$id"};
3064     return if !defined($audio);
3065
3066     my $audioproperties = parse_property_string($audio_fmt, $audio);
3067     my $audiodriver = $audioproperties->{driver} // 'spice';
3068
3069     return {
3070         dev => $audioproperties->{device},
3071         dev_id => "audiodev$id",
3072         backend => $audiodriver,
3073         backend_id => "$audiodriver-backend${id}",
3074     };
3075 }
3076
3077 sub audio_devs {
3078     my ($audio, $audiopciaddr, $machine_version) = @_;
3079
3080     my $devs = [];
3081
3082     my $id = $audio->{dev_id};
3083     my $audiodev = "";
3084     if (min_version($machine_version, 4, 2)) {
3085         $audiodev = ",audiodev=$audio->{backend_id}";
3086     }
3087
3088     if ($audio->{dev} eq 'AC97') {
3089         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3090     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3091         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3092         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3093         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3094     } else {
3095         die "unkown audio device '$audio->{dev}', implement me!";
3096     }
3097
3098     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3099
3100     return $devs;
3101 }
3102
3103 sub get_tpm_paths {
3104     my ($vmid) = @_;
3105     return {
3106         socket => "/var/run/qemu-server/$vmid.swtpm",
3107         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3108     };
3109 }
3110
3111 sub add_tpm_device {
3112     my ($vmid, $devices, $conf) = @_;
3113
3114     return if !$conf->{tpmstate0};
3115
3116     my $paths = get_tpm_paths($vmid);
3117
3118     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3119     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3120     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3121 }
3122
3123 sub start_swtpm {
3124     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3125
3126     return if !$tpmdrive;
3127
3128     my $state;
3129     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3130     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3131     if ($storeid) {
3132         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3133     } else {
3134         $state = $tpm->{file};
3135     }
3136
3137     my $paths = get_tpm_paths($vmid);
3138
3139     # during migration, we will get state from remote
3140     #
3141     if (!$migration) {
3142         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3143         my $setup_cmd = [
3144             "swtpm_setup",
3145             "--tpmstate",
3146             "file://$state",
3147             "--createek",
3148             "--create-ek-cert",
3149             "--create-platform-cert",
3150             "--lock-nvram",
3151             "--config",
3152             "/etc/swtpm_setup.conf", # do not use XDG configs
3153             "--runas",
3154             "0", # force creation as root, error if not possible
3155             "--not-overwrite", # ignore existing state, do not modify
3156         ];
3157
3158         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3159         # TPM 2.0 supports ECC crypto, use if possible
3160         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3161
3162         run_command($setup_cmd, outfunc => sub {
3163             print "swtpm_setup: $1\n";
3164         });
3165     }
3166
3167     # Used to distinguish different invocations in the log.
3168     my $log_prefix = "[id=" . int(time()) . "] ";
3169
3170     my $emulator_cmd = [
3171         "swtpm",
3172         "socket",
3173         "--tpmstate",
3174         "backend-uri=file://$state,mode=0600",
3175         "--ctrl",
3176         "type=unixio,path=$paths->{socket},mode=0600",
3177         "--pid",
3178         "file=$paths->{pid}",
3179         "--terminate", # terminate on QEMU disconnect
3180         "--daemon",
3181         "--log",
3182         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3183     ];
3184     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3185     run_command($emulator_cmd, outfunc => sub { print $1; });
3186
3187     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3188     while (! -e $paths->{pid}) {
3189         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3190         usleep(50_000);
3191     }
3192
3193     # return untainted PID of swtpm daemon so it can be killed on error
3194     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3195     return $1;
3196 }
3197
3198 sub vga_conf_has_spice {
3199     my ($vga) = @_;
3200
3201     my $vgaconf = parse_vga($vga);
3202     my $vgatype = $vgaconf->{type};
3203     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3204
3205     return $1 || 1;
3206 }
3207
3208 sub is_native($) {
3209     my ($arch) = @_;
3210     return get_host_arch() eq $arch;
3211 }
3212
3213 sub get_vm_arch {
3214     my ($conf) = @_;
3215     return $conf->{arch} // get_host_arch();
3216 }
3217
3218 my $default_machines = {
3219     x86_64 => 'pc',
3220     aarch64 => 'virt',
3221 };
3222
3223 sub get_installed_machine_version {
3224     my ($kvmversion) = @_;
3225     $kvmversion = kvm_user_version() if !defined($kvmversion);
3226     $kvmversion =~ m/^(\d+\.\d+)/;
3227     return $1;
3228 }
3229
3230 sub windows_get_pinned_machine_version {
3231     my ($machine, $base_version, $kvmversion) = @_;
3232
3233     my $pin_version = $base_version;
3234     if (!defined($base_version) ||
3235         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3236     ) {
3237         $pin_version = get_installed_machine_version($kvmversion);
3238     }
3239     if (!$machine || $machine eq 'pc') {
3240         $machine = "pc-i440fx-$pin_version";
3241     } elsif ($machine eq 'q35') {
3242         $machine = "pc-q35-$pin_version";
3243     } elsif ($machine eq 'virt') {
3244         $machine = "virt-$pin_version";
3245     } else {
3246         warn "unknown machine type '$machine', not touching that!\n";
3247     }
3248
3249     return $machine;
3250 }
3251
3252 sub get_vm_machine {
3253     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3254
3255     my $machine = $forcemachine || $conf->{machine};
3256
3257     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3258         $kvmversion //= kvm_user_version();
3259         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3260         # layout which confuses windows quite a bit and may result in various regressions..
3261         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3262         if (windows_version($conf->{ostype})) {
3263             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3264         }
3265         $arch //= 'x86_64';
3266         $machine ||= $default_machines->{$arch};
3267         if ($add_pve_version) {
3268             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3269             $machine .= "+pve$pvever";
3270         }
3271     }
3272
3273     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3274         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3275         $machine = $1 if $is_pxe;
3276
3277         # for version-pinned machines that do not include a pve-version (e.g.
3278         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3279         $machine .= '+pve0';
3280
3281         $machine .= '.pxe' if $is_pxe;
3282     }
3283
3284     return $machine;
3285 }
3286
3287 sub get_ovmf_files($$$) {
3288     my ($arch, $efidisk, $smm) = @_;
3289
3290     my $types = $OVMF->{$arch}
3291         or die "no OVMF images known for architecture '$arch'\n";
3292
3293     my $type = 'default';
3294     if ($arch eq 'x86_64') {
3295         if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3296             $type = $smm ? "4m" : "4m-no-smm";
3297             $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3298         } else {
3299             # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3300         }
3301     }
3302
3303     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3304     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3305     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3306
3307     return ($ovmf_code, $ovmf_vars);
3308 }
3309
3310 my $Arch2Qemu = {
3311     aarch64 => '/usr/bin/qemu-system-aarch64',
3312     x86_64 => '/usr/bin/qemu-system-x86_64',
3313 };
3314 sub get_command_for_arch($) {
3315     my ($arch) = @_;
3316     return '/usr/bin/kvm' if is_native($arch);
3317
3318     my $cmd = $Arch2Qemu->{$arch}
3319         or die "don't know how to emulate architecture '$arch'\n";
3320     return $cmd;
3321 }
3322
3323 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3324 # to use in a QEMU command line (-cpu element), first array_intersect the result
3325 # of query_supported_ with query_understood_. This is necessary because:
3326 #
3327 # a) query_understood_ returns flags the host cannot use and
3328 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3329 #    flags, but CPU settings - with most of them being flags. Those settings
3330 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3331 #
3332 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3333 # expensive. If you need the value returned from this, you can get it much
3334 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3335 # $accel being 'kvm' or 'tcg'.
3336 #
3337 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3338 # changes, automatically populating pmxcfs.
3339 #
3340 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3341 # since kvm and tcg machines support different flags
3342 #
3343 sub query_supported_cpu_flags {
3344     my ($arch) = @_;
3345
3346     $arch //= get_host_arch();
3347     my $default_machine = $default_machines->{$arch};
3348
3349     my $flags = {};
3350
3351     # FIXME: Once this is merged, the code below should work for ARM as well:
3352     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3353     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3354         $arch eq "aarch64";
3355
3356     my $kvm_supported = defined(kvm_version());
3357     my $qemu_cmd = get_command_for_arch($arch);
3358     my $fakevmid = -1;
3359     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3360
3361     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3362     my $query_supported_run_qemu = sub {
3363         my ($kvm) = @_;
3364
3365         my $flags = {};
3366         my $cmd = [
3367             $qemu_cmd,
3368             '-machine', $default_machine,
3369             '-display', 'none',
3370             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3371             '-mon', 'chardev=qmp,mode=control',
3372             '-pidfile', $pidfile,
3373             '-S', '-daemonize'
3374         ];
3375
3376         if (!$kvm) {
3377             push @$cmd, '-accel', 'tcg';
3378         }
3379
3380         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3381         die "QEMU flag querying VM exited with code " . $rc if $rc;
3382
3383         eval {
3384             my $cmd_result = mon_cmd(
3385                 $fakevmid,
3386                 'query-cpu-model-expansion',
3387                 type => 'full',
3388                 model => { name => 'host' }
3389             );
3390
3391             my $props = $cmd_result->{model}->{props};
3392             foreach my $prop (keys %$props) {
3393                 next if $props->{$prop} ne '1';
3394                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3395                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3396                 # We only keep those with underscores, to match /proc/cpuinfo
3397                 $prop =~ s/\.|-/_/g;
3398                 $flags->{$prop} = 1;
3399             }
3400         };
3401         my $err = $@;
3402
3403         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3404         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3405
3406         die $err if $err;
3407
3408         return [ sort keys %$flags ];
3409     };
3410
3411     # We need to query QEMU twice, since KVM and TCG have different supported flags
3412     PVE::QemuConfig->lock_config($fakevmid, sub {
3413         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3414         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3415
3416         if ($kvm_supported) {
3417             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3418             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3419         }
3420     });
3421
3422     return $flags;
3423 }
3424
3425 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3426 my $understood_cpu_flag_dir = "/usr/share/kvm";
3427 sub query_understood_cpu_flags {
3428     my $arch = get_host_arch();
3429     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3430
3431     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3432         if ! -e $filepath;
3433
3434     my $raw = file_get_contents($filepath);
3435     $raw =~ s/^\s+|\s+$//g;
3436     my @flags = split(/\s+/, $raw);
3437
3438     return \@flags;
3439 }
3440
3441 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3442 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3443 my sub should_disable_smm {
3444     my ($conf, $vga, $machine) = @_;
3445
3446     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3447
3448     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3449         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3450 }
3451
3452 my sub print_ovmf_drive_commandlines {
3453     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3454
3455     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3456
3457     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3458
3459     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3460     if ($d) {
3461         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3462         my ($path, $format) = $d->@{'file', 'format'};
3463         if ($storeid) {
3464             $path = PVE::Storage::path($storecfg, $d->{file});
3465             if (!defined($format)) {
3466                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3467                 $format = qemu_img_format($scfg, $volname);
3468             }
3469         } elsif (!defined($format)) {
3470             die "efidisk format must be specified\n";
3471         }
3472         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3473         if ($path =~ m/^rbd:/) {
3474             $var_drive_str .= ',cache=writeback';
3475             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3476         }
3477         $var_drive_str .= ",format=$format,file=$path";
3478
3479         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3480         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3481     } else {
3482         log_warn("no efidisk configured! Using temporary efivars disk.");
3483         my $path = "/tmp/$vmid-ovmf.fd";
3484         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3485         $var_drive_str .= ",format=raw,file=$path";
3486         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3487     }
3488
3489     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3490 }
3491
3492 sub config_to_command {
3493     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3494         $pbs_backing) = @_;
3495
3496     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3497     my $devices = [];
3498     my $bridges = {};
3499     my $ostype = $conf->{ostype};
3500     my $winversion = windows_version($ostype);
3501     my $kvm = $conf->{kvm};
3502     my $nodename = nodename();
3503
3504     my $arch = get_vm_arch($conf);
3505     my $kvm_binary = get_command_for_arch($arch);
3506     my $kvmver = kvm_user_version($kvm_binary);
3507
3508     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3509         $kvmver //= "undefined";
3510         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3511     }
3512
3513     my $add_pve_version = min_version($kvmver, 4, 1);
3514
3515     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3516     my $machine_version = extract_version($machine_type, $kvmver);
3517     $kvm //= 1 if is_native($arch);
3518
3519     $machine_version =~ m/(\d+)\.(\d+)/;
3520     my ($machine_major, $machine_minor) = ($1, $2);
3521
3522     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3523         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3524     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3525         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3526             ." please upgrade node '$nodename'\n"
3527     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3528         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3529         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3530             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3531             ." node '$nodename'\n";
3532     }
3533
3534     # if a specific +pve version is required for a feature, use $version_guard
3535     # instead of min_version to allow machines to be run with the minimum
3536     # required version
3537     my $required_pve_version = 0;
3538     my $version_guard = sub {
3539         my ($major, $minor, $pve) = @_;
3540         return 0 if !min_version($machine_version, $major, $minor, $pve);
3541         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3542         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3543         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3544         return 1;
3545     };
3546
3547     if ($kvm && !defined kvm_version()) {
3548         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3549             ." or enable in BIOS.\n";
3550     }
3551
3552     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3553     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3554     my $use_old_bios_files = undef;
3555     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3556
3557     my $cmd = [];
3558     if ($conf->{affinity}) {
3559         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3560     }
3561
3562     push @$cmd, $kvm_binary;
3563
3564     push @$cmd, '-id', $vmid;
3565
3566     my $vmname = $conf->{name} || "vm$vmid";
3567
3568     push @$cmd, '-name', "$vmname,debug-threads=on";
3569
3570     push @$cmd, '-no-shutdown';
3571
3572     my $use_virtio = 0;
3573
3574     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3575     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3576     push @$cmd, '-mon', "chardev=qmp,mode=control";
3577
3578     if (min_version($machine_version, 2, 12)) {
3579         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3580         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3581     }
3582
3583     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3584
3585     push @$cmd, '-daemonize';
3586
3587     if ($conf->{smbios1}) {
3588         my $smbios_conf = parse_smbios1($conf->{smbios1});
3589         if ($smbios_conf->{base64}) {
3590             # Do not pass base64 flag to qemu
3591             delete $smbios_conf->{base64};
3592             my $smbios_string = "";
3593             foreach my $key (keys %$smbios_conf) {
3594                 my $value;
3595                 if ($key eq "uuid") {
3596                     $value = $smbios_conf->{uuid}
3597                 } else {
3598                     $value = decode_base64($smbios_conf->{$key});
3599                 }
3600                 # qemu accepts any binary data, only commas need escaping by double comma
3601                 $value =~ s/,/,,/g;
3602                 $smbios_string .= "," . $key . "=" . $value if $value;
3603             }
3604             push @$cmd, '-smbios', "type=1" . $smbios_string;
3605         } else {
3606             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3607         }
3608     }
3609
3610     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3611         my ($code_drive_str, $var_drive_str) =
3612             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3613         push $cmd->@*, '-drive', $code_drive_str;
3614         push $cmd->@*, '-drive', $var_drive_str;
3615     }
3616
3617     if ($q35) { # tell QEMU to load q35 config early
3618         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3619         if (min_version($machine_version, 4, 0)) {
3620             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3621         } else {
3622             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3623         }
3624     }
3625
3626     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3627         push @$cmd, $fixups->@*;
3628     }
3629
3630     if ($conf->{vmgenid}) {
3631         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3632     }
3633
3634     # add usb controllers
3635     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3636         $conf, $bridges, $arch, $machine_type, $machine_version);
3637     push @$devices, @usbcontrollers if @usbcontrollers;
3638     my $vga = parse_vga($conf->{vga});
3639
3640     my $qxlnum = vga_conf_has_spice($conf->{vga});
3641     $vga->{type} = 'qxl' if $qxlnum;
3642
3643     if (!$vga->{type}) {
3644         if ($arch eq 'aarch64') {
3645             $vga->{type} = 'virtio';
3646         } elsif (min_version($machine_version, 2, 9)) {
3647             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3648         } else {
3649             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3650         }
3651     }
3652
3653     # enable absolute mouse coordinates (needed by vnc)
3654     my $tablet = $conf->{tablet};
3655     if (!defined($tablet)) {
3656         $tablet = $defaults->{tablet};
3657         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3658         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3659     }
3660
3661     if ($tablet) {
3662         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3663         my $kbd = print_keyboarddevice_full($conf, $arch);
3664         push @$devices, '-device', $kbd if defined($kbd);
3665     }
3666
3667     my $bootorder = device_bootorder($conf);
3668
3669     # host pci device passthrough
3670     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3671         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3672
3673     # usb devices
3674     my $usb_dev_features = {};
3675     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3676
3677     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3678         $conf, $usb_dev_features, $bootorder, $machine_version);
3679     push @$devices, @usbdevices if @usbdevices;
3680
3681     # serial devices
3682     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3683         my $path = $conf->{"serial$i"} or next;
3684         if ($path eq 'socket') {
3685             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3686             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3687             # On aarch64, serial0 is the UART device. QEMU only allows
3688             # connecting UART devices via the '-serial' command line, as
3689             # the device has a fixed slot on the hardware...
3690             if ($arch eq 'aarch64' && $i == 0) {
3691                 push @$devices, '-serial', "chardev:serial$i";
3692             } else {
3693                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3694             }
3695         } else {
3696             die "no such serial device\n" if ! -c $path;
3697             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3698             push @$devices, '-device', "isa-serial,chardev=serial$i";
3699         }
3700     }
3701
3702     # parallel devices
3703     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3704         if (my $path = $conf->{"parallel$i"}) {
3705             die "no such parallel device\n" if ! -c $path;
3706             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3707             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3708             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3709         }
3710     }
3711
3712     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3713         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3714         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3715         push @$devices, @$audio_devs;
3716     }
3717
3718     # Add a TPM only if the VM is not a template,
3719     # to support backing up template VMs even if the TPM disk is write-protected.
3720     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3721
3722     my $sockets = 1;
3723     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3724     $sockets = $conf->{sockets} if  $conf->{sockets};
3725
3726     my $cores = $conf->{cores} || 1;
3727
3728     my $maxcpus = $sockets * $cores;
3729
3730     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3731
3732     my $allowed_vcpus = $cpuinfo->{cpus};
3733
3734     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3735
3736     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3737         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3738         for (my $i = 2; $i <= $vcpus; $i++)  {
3739             my $cpustr = print_cpu_device($conf,$i);
3740             push @$cmd, '-device', $cpustr;
3741         }
3742
3743     } else {
3744
3745         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3746     }
3747     push @$cmd, '-nodefaults';
3748
3749     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3750
3751     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3752
3753     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3754
3755     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3756         push @$devices, '-device', print_vga_device(
3757             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3758
3759         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3760
3761         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3762         push @$cmd,  '-vnc', "unix:$socket,password=on";
3763     } else {
3764         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3765         push @$cmd, '-nographic';
3766     }
3767
3768     # time drift fix
3769     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3770     my $useLocaltime = $conf->{localtime};
3771
3772     if ($winversion >= 5) { # windows
3773         $useLocaltime = 1 if !defined($conf->{localtime});
3774
3775         # use time drift fix when acpi is enabled
3776         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3777             $tdf = 1 if !defined($conf->{tdf});
3778         }
3779     }
3780
3781     if ($winversion >= 6) {
3782         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3783         push @$machineFlags, 'hpet=off';
3784     }
3785
3786     push @$rtcFlags, 'driftfix=slew' if $tdf;
3787
3788     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3789         push @$rtcFlags, "base=$conf->{startdate}";
3790     } elsif ($useLocaltime) {
3791         push @$rtcFlags, 'base=localtime';
3792     }
3793
3794     if ($forcecpu) {
3795         push @$cmd, '-cpu', $forcecpu;
3796     } else {
3797         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3798     }
3799
3800     PVE::QemuServer::Memory::config(
3801         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3802
3803     push @$cmd, '-S' if $conf->{freeze};
3804
3805     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3806
3807     my $guest_agent = parse_guest_agent($conf);
3808
3809     if ($guest_agent->{enabled}) {
3810         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3811         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3812
3813         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3814             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3815             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3816             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3817         } elsif ($guest_agent->{type} eq 'isa') {
3818             push @$devices, '-device', "isa-serial,chardev=qga0";
3819         }
3820     }
3821
3822     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3823     if ($rng && $version_guard->(4, 1, 2)) {
3824         check_rng_source($rng->{source});
3825
3826         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3827         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3828         my $limiter_str = "";
3829         if ($max_bytes) {
3830             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3831         }
3832
3833         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3834         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3835         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3836     }
3837
3838     my $spice_port;
3839
3840     assert_clipboard_config($vga);
3841     my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3842
3843     if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3844         if ($qxlnum > 1) {
3845             if ($winversion){
3846                 for (my $i = 1; $i < $qxlnum; $i++){
3847                     push @$devices, '-device', print_vga_device(
3848                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3849                 }
3850             } else {
3851                 # assume other OS works like Linux
3852                 my ($ram, $vram) = ("134217728", "67108864");
3853                 if ($vga->{memory}) {
3854                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3855                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3856                 }
3857                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3858                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3859             }
3860         }
3861
3862         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3863
3864         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3865         if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3866             push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3867         } else {
3868             push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3869         }
3870         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3871
3872         if ($is_spice) {
3873             my $pfamily = PVE::Tools::get_host_address_family($nodename);
3874             my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3875             die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3876
3877             my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3878             $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3879
3880             my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3881             my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3882             if ($spice_enhancement->{foldersharing}) {
3883                 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3884                 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3885             }
3886
3887             my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3888             $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3889                 if $spice_enhancement->{videostreaming};
3890             push @$devices, '-spice', "$spice_opts";
3891         }
3892     }
3893
3894     # enable balloon by default, unless explicitly disabled
3895     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3896         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3897         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3898         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3899         push @$devices, '-device', $ballooncmd;
3900     }
3901
3902     if ($conf->{watchdog}) {
3903         my $wdopts = parse_watchdog($conf->{watchdog});
3904         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3905         my $watchdog = $wdopts->{model} || 'i6300esb';
3906         push @$devices, '-device', "$watchdog$pciaddr";
3907         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3908     }
3909
3910     my $vollist = [];
3911     my $scsicontroller = {};
3912     my $ahcicontroller = {};
3913     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3914
3915     # Add iscsi initiator name if available
3916     if (my $initiator = get_initiator_name()) {
3917         push @$devices, '-iscsi', "initiator-name=$initiator";
3918     }
3919
3920     PVE::QemuConfig->foreach_volume($conf, sub {
3921         my ($ds, $drive) = @_;
3922
3923         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3924             check_volume_storage_type($storecfg, $drive->{file});
3925             push @$vollist, $drive->{file};
3926         }
3927
3928         # ignore efidisk here, already added in bios/fw handling code above
3929         return if $drive->{interface} eq 'efidisk';
3930         # similar for TPM
3931         return if $drive->{interface} eq 'tpmstate';
3932
3933         $use_virtio = 1 if $ds =~ m/^virtio/;
3934
3935         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3936
3937         if ($drive->{interface} eq 'virtio'){
3938            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3939         }
3940
3941         if ($drive->{interface} eq 'scsi') {
3942
3943             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3944
3945             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3946                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3947
3948             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3949             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3950
3951             my $iothread = '';
3952             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3953                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3954                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3955             } elsif ($drive->{iothread}) {
3956                 log_warn(
3957                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3958                 );
3959             }
3960
3961             my $queues = '';
3962             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3963                 $queues = ",num_queues=$drive->{queues}";
3964             }
3965
3966             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3967                 if !$scsicontroller->{$controller};
3968             $scsicontroller->{$controller}=1;
3969         }
3970
3971         if ($drive->{interface} eq 'sata') {
3972             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3973             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3974             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3975                 if !$ahcicontroller->{$controller};
3976             $ahcicontroller->{$controller}=1;
3977         }
3978
3979         my $pbs_conf = $pbs_backing->{$ds};
3980         my $pbs_name = undef;
3981         if ($pbs_conf) {
3982             $pbs_name = "drive-$ds-pbs";
3983             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3984         }
3985
3986         my $drive_cmd = print_drive_commandline_full(
3987             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3988
3989         # extra protection for templates, but SATA and IDE don't support it..
3990         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3991
3992         push @$devices, '-drive',$drive_cmd;
3993         push @$devices, '-device', print_drivedevice_full(
3994             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3995     });
3996
3997     for (my $i = 0; $i < $MAX_NETS; $i++) {
3998         my $netname = "net$i";
3999
4000         next if !$conf->{$netname};
4001         my $d = parse_net($conf->{$netname});
4002         next if !$d;
4003         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4004
4005         $use_virtio = 1 if $d->{model} eq 'virtio';
4006
4007         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4008
4009         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4010         push @$devices, '-netdev', $netdevfull;
4011
4012         my $netdevicefull = print_netdevice_full(
4013             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4014
4015         push @$devices, '-device', $netdevicefull;
4016     }
4017
4018     if ($conf->{ivshmem}) {
4019         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4020
4021         my $bus;
4022         if ($q35) {
4023             $bus = print_pcie_addr("ivshmem");
4024         } else {
4025             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4026         }
4027
4028         my $ivshmem_name = $ivshmem->{name} // $vmid;
4029         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4030
4031         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4032         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4033             .",size=$ivshmem->{size}M";
4034     }
4035
4036     # pci.4 is nested in pci.1
4037     $bridges->{1} = 1 if $bridges->{4};
4038
4039     if (!$q35) { # add pci bridges
4040         if (min_version($machine_version, 2, 3)) {
4041            $bridges->{1} = 1;
4042            $bridges->{2} = 1;
4043         }
4044         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4045     }
4046
4047     for my $k (sort {$b cmp $a} keys %$bridges) {
4048         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4049
4050         my $k_name = $k;
4051         if ($k == 2 && $legacy_igd) {
4052             $k_name = "$k-igd";
4053         }
4054         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4055         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4056
4057         if ($q35) { # add after -readconfig pve-q35.cfg
4058             splice @$devices, 2, 0, '-device', $devstr;
4059         } else {
4060             unshift @$devices, '-device', $devstr if $k > 0;
4061         }
4062     }
4063
4064     if (!$kvm) {
4065         push @$machineFlags, 'accel=tcg';
4066     }
4067
4068     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4069
4070     my $machine_type_min = $machine_type;
4071     if ($add_pve_version) {
4072         $machine_type_min =~ s/\+pve\d+$//;
4073         $machine_type_min .= "+pve$required_pve_version";
4074     }
4075     push @$machineFlags, "type=${machine_type_min}";
4076
4077     push @$cmd, @$devices;
4078     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4079     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4080     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4081
4082     if (my $vmstate = $conf->{vmstate}) {
4083         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4084         push @$vollist, $vmstate;
4085         push @$cmd, '-loadstate', $statepath;
4086         print "activating and using '$vmstate' as vmstate\n";
4087     }
4088
4089     if (PVE::QemuConfig->is_template($conf)) {
4090         # needed to workaround base volumes being read-only
4091         push @$cmd, '-snapshot';
4092     }
4093
4094     # add custom args
4095     if ($conf->{args}) {
4096         my $aa = PVE::Tools::split_args($conf->{args});
4097         push @$cmd, @$aa;
4098     }
4099
4100     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4101 }
4102
4103 sub check_rng_source {
4104     my ($source) = @_;
4105
4106     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4107     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4108         if ! -e $source;
4109
4110     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4111     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4112         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4113         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4114         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4115             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4116             ." to the host.\n";
4117     }
4118 }
4119
4120 sub spice_port {
4121     my ($vmid) = @_;
4122
4123     my $res = mon_cmd($vmid, 'query-spice');
4124
4125     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4126 }
4127
4128 sub vm_devices_list {
4129     my ($vmid) = @_;
4130
4131     my $res = mon_cmd($vmid, 'query-pci');
4132     my $devices_to_check = [];
4133     my $devices = {};
4134     foreach my $pcibus (@$res) {
4135         push @$devices_to_check, @{$pcibus->{devices}},
4136     }
4137
4138     while (@$devices_to_check) {
4139         my $to_check = [];
4140         for my $d (@$devices_to_check) {
4141             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4142             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4143
4144             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4145             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4146         }
4147         $devices_to_check = $to_check;
4148     }
4149
4150     my $resblock = mon_cmd($vmid, 'query-block');
4151     foreach my $block (@$resblock) {
4152         if($block->{device} =~ m/^drive-(\S+)/){
4153                 $devices->{$1} = 1;
4154         }
4155     }
4156
4157     my $resmice = mon_cmd($vmid, 'query-mice');
4158     foreach my $mice (@$resmice) {
4159         if ($mice->{name} eq 'QEMU HID Tablet') {
4160             $devices->{tablet} = 1;
4161             last;
4162         }
4163     }
4164
4165     # for usb devices there is no query-usb
4166     # but we can iterate over the entries in
4167     # qom-list path=/machine/peripheral
4168     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4169     foreach my $per (@$resperipheral) {
4170         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4171             $devices->{$per->{name}} = 1;
4172         }
4173     }
4174
4175     return $devices;
4176 }
4177
4178 sub vm_deviceplug {
4179     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4180
4181     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4182
4183     my $devices_list = vm_devices_list($vmid);
4184     return 1 if defined($devices_list->{$deviceid});
4185
4186     # add PCI bridge if we need it for the device
4187     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4188
4189     if ($deviceid eq 'tablet') {
4190         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4191     } elsif ($deviceid eq 'keyboard') {
4192         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4193     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4194         my $id = $1;
4195         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4196         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4197     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4198         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4199     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4200         qemu_iothread_add($vmid, $deviceid, $device);
4201
4202         qemu_driveadd($storecfg, $vmid, $device);
4203         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4204
4205         qemu_deviceadd($vmid, $devicefull);
4206         eval { qemu_deviceaddverify($vmid, $deviceid); };
4207         if (my $err = $@) {
4208             eval { qemu_drivedel($vmid, $deviceid); };
4209             warn $@ if $@;
4210             die $err;
4211         }
4212     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4213         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4214         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4215         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4216
4217         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4218
4219         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4220             qemu_iothread_add($vmid, $deviceid, $device);
4221             $devicefull .= ",iothread=iothread-$deviceid";
4222         }
4223
4224         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4225             $devicefull .= ",num_queues=$device->{queues}";
4226         }
4227
4228         qemu_deviceadd($vmid, $devicefull);
4229         qemu_deviceaddverify($vmid, $deviceid);
4230     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4231         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4232         qemu_driveadd($storecfg, $vmid, $device);
4233
4234         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4235         eval { qemu_deviceadd($vmid, $devicefull); };
4236         if (my $err = $@) {
4237             eval { qemu_drivedel($vmid, $deviceid); };
4238             warn $@ if $@;
4239             die $err;
4240         }
4241     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4242         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4243
4244         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4245         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4246         my $use_old_bios_files = undef;
4247         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4248
4249         my $netdevicefull = print_netdevice_full(
4250             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4251         qemu_deviceadd($vmid, $netdevicefull);
4252         eval {
4253             qemu_deviceaddverify($vmid, $deviceid);
4254             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4255         };
4256         if (my $err = $@) {
4257             eval { qemu_netdevdel($vmid, $deviceid); };
4258             warn $@ if $@;
4259             die $err;
4260         }
4261     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4262         my $bridgeid = $2;
4263         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4264         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4265
4266         qemu_deviceadd($vmid, $devicefull);
4267         qemu_deviceaddverify($vmid, $deviceid);
4268     } else {
4269         die "can't hotplug device '$deviceid'\n";
4270     }
4271
4272     return 1;
4273 }
4274
4275 # fixme: this should raise exceptions on error!
4276 sub vm_deviceunplug {
4277     my ($vmid, $conf, $deviceid) = @_;
4278
4279     my $devices_list = vm_devices_list($vmid);
4280     return 1 if !defined($devices_list->{$deviceid});
4281
4282     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4283     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4284
4285     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4286         qemu_devicedel($vmid, $deviceid);
4287     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4288         qemu_devicedel($vmid, $deviceid);
4289         qemu_devicedelverify($vmid, $deviceid);
4290     } elsif ($deviceid =~ m/^usb\d+$/) {
4291         qemu_devicedel($vmid, $deviceid);
4292         qemu_devicedelverify($vmid, $deviceid);
4293     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4294         my $device = parse_drive($deviceid, $conf->{$deviceid});
4295
4296         qemu_devicedel($vmid, $deviceid);
4297         qemu_devicedelverify($vmid, $deviceid);
4298         qemu_drivedel($vmid, $deviceid);
4299         qemu_iothread_del($vmid, $deviceid, $device);
4300     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4301         qemu_devicedel($vmid, $deviceid);
4302         qemu_devicedelverify($vmid, $deviceid);
4303     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4304         my $device = parse_drive($deviceid, $conf->{$deviceid});
4305
4306         qemu_devicedel($vmid, $deviceid);
4307         qemu_devicedelverify($vmid, $deviceid);
4308         qemu_drivedel($vmid, $deviceid);
4309         qemu_deletescsihw($conf, $vmid, $deviceid);
4310
4311         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4312             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4313     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4314         qemu_devicedel($vmid, $deviceid);
4315         qemu_devicedelverify($vmid, $deviceid);
4316         qemu_netdevdel($vmid, $deviceid);
4317     } else {
4318         die "can't unplug device '$deviceid'\n";
4319     }
4320
4321     return 1;
4322 }
4323
4324 sub qemu_spice_usbredir_chardev_add {
4325     my ($vmid, $id) = @_;
4326
4327     mon_cmd($vmid, "chardev-add" , (
4328         id => $id,
4329         backend => {
4330             type => 'spicevmc',
4331             data => {
4332                 type => "usbredir",
4333             },
4334         },
4335     ));
4336 }
4337
4338 sub qemu_iothread_add {
4339     my ($vmid, $deviceid, $device) = @_;
4340
4341     if ($device->{iothread}) {
4342         my $iothreads = vm_iothreads_list($vmid);
4343         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4344     }
4345 }
4346
4347 sub qemu_iothread_del {
4348     my ($vmid, $deviceid, $device) = @_;
4349
4350     if ($device->{iothread}) {
4351         my $iothreads = vm_iothreads_list($vmid);
4352         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4353     }
4354 }
4355
4356 sub qemu_driveadd {
4357     my ($storecfg, $vmid, $device) = @_;
4358
4359     my $kvmver = get_running_qemu_version($vmid);
4360     my $io_uring = min_version($kvmver, 6, 0);
4361     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4362     $drive =~ s/\\/\\\\/g;
4363     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4364
4365     # If the command succeeds qemu prints: "OK"
4366     return 1 if $ret =~ m/OK/s;
4367
4368     die "adding drive failed: $ret\n";
4369 }
4370
4371 sub qemu_drivedel {
4372     my ($vmid, $deviceid) = @_;
4373
4374     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4375     $ret =~ s/^\s+//;
4376
4377     return 1 if $ret eq "";
4378
4379     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4380     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4381
4382     die "deleting drive $deviceid failed : $ret\n";
4383 }
4384
4385 sub qemu_deviceaddverify {
4386     my ($vmid, $deviceid) = @_;
4387
4388     for (my $i = 0; $i <= 5; $i++) {
4389          my $devices_list = vm_devices_list($vmid);
4390          return 1 if defined($devices_list->{$deviceid});
4391          sleep 1;
4392     }
4393
4394     die "error on hotplug device '$deviceid'\n";
4395 }
4396
4397
4398 sub qemu_devicedelverify {
4399     my ($vmid, $deviceid) = @_;
4400
4401     # need to verify that the device is correctly removed as device_del
4402     # is async and empty return is not reliable
4403
4404     for (my $i = 0; $i <= 5; $i++) {
4405          my $devices_list = vm_devices_list($vmid);
4406          return 1 if !defined($devices_list->{$deviceid});
4407          sleep 1;
4408     }
4409
4410     die "error on hot-unplugging device '$deviceid'\n";
4411 }
4412
4413 sub qemu_findorcreatescsihw {
4414     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4415
4416     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4417
4418     my $scsihwid="$controller_prefix$controller";
4419     my $devices_list = vm_devices_list($vmid);
4420
4421     if (!defined($devices_list->{$scsihwid})) {
4422         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4423     }
4424
4425     return 1;
4426 }
4427
4428 sub qemu_deletescsihw {
4429     my ($conf, $vmid, $opt) = @_;
4430
4431     my $device = parse_drive($opt, $conf->{$opt});
4432
4433     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4434         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4435         return 1;
4436     }
4437
4438     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4439
4440     my $devices_list = vm_devices_list($vmid);
4441     foreach my $opt (keys %{$devices_list}) {
4442         if (is_valid_drivename($opt)) {
4443             my $drive = parse_drive($opt, $conf->{$opt});
4444             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4445                 return 1;
4446             }
4447         }
4448     }
4449
4450     my $scsihwid="scsihw$controller";
4451
4452     vm_deviceunplug($vmid, $conf, $scsihwid);
4453
4454     return 1;
4455 }
4456
4457 sub qemu_add_pci_bridge {
4458     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4459
4460     my $bridges = {};
4461
4462     my $bridgeid;
4463
4464     print_pci_addr($device, $bridges, $arch, $machine_type);
4465
4466     while (my ($k, $v) = each %$bridges) {
4467         $bridgeid = $k;
4468     }
4469     return 1 if !defined($bridgeid) || $bridgeid < 1;
4470
4471     my $bridge = "pci.$bridgeid";
4472     my $devices_list = vm_devices_list($vmid);
4473
4474     if (!defined($devices_list->{$bridge})) {
4475         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4476     }
4477
4478     return 1;
4479 }
4480
4481 sub qemu_set_link_status {
4482     my ($vmid, $device, $up) = @_;
4483
4484     mon_cmd($vmid, "set_link", name => $device,
4485                up => $up ? JSON::true : JSON::false);
4486 }
4487
4488 sub qemu_netdevadd {
4489     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4490
4491     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4492     my %options =  split(/[=,]/, $netdev);
4493
4494     if (defined(my $vhost = $options{vhost})) {
4495         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4496     }
4497
4498     if (defined(my $queues = $options{queues})) {
4499         $options{queues} = $queues + 0;
4500     }
4501
4502     mon_cmd($vmid, "netdev_add",  %options);
4503     return 1;
4504 }
4505
4506 sub qemu_netdevdel {
4507     my ($vmid, $deviceid) = @_;
4508
4509     mon_cmd($vmid, "netdev_del", id => $deviceid);
4510 }
4511
4512 sub qemu_usb_hotplug {
4513     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4514
4515     return if !$device;
4516
4517     # remove the old one first
4518     vm_deviceunplug($vmid, $conf, $deviceid);
4519
4520     # check if xhci controller is necessary and available
4521     my $devicelist = vm_devices_list($vmid);
4522
4523     if (!$devicelist->{xhci}) {
4524         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4525         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4526     }
4527
4528     # add the new one
4529     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4530 }
4531
4532 sub qemu_cpu_hotplug {
4533     my ($vmid, $conf, $vcpus) = @_;
4534
4535     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4536
4537     my $sockets = 1;
4538     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4539     $sockets = $conf->{sockets} if  $conf->{sockets};
4540     my $cores = $conf->{cores} || 1;
4541     my $maxcpus = $sockets * $cores;
4542
4543     $vcpus = $maxcpus if !$vcpus;
4544
4545     die "you can't add more vcpus than maxcpus\n"
4546         if $vcpus > $maxcpus;
4547
4548     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4549
4550     if ($vcpus < $currentvcpus) {
4551
4552         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4553
4554             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4555                 qemu_devicedel($vmid, "cpu$i");
4556                 my $retry = 0;
4557                 my $currentrunningvcpus = undef;
4558                 while (1) {
4559                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4560                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4561                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4562                     $retry++;
4563                     sleep 1;
4564                 }
4565                 #update conf after each succesfull cpu unplug
4566                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4567                 PVE::QemuConfig->write_config($vmid, $conf);
4568             }
4569         } else {
4570             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4571         }
4572
4573         return;
4574     }
4575
4576     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4577     die "vcpus in running vm does not match its configuration\n"
4578         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4579
4580     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4581
4582         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4583             my $cpustr = print_cpu_device($conf, $i);
4584             qemu_deviceadd($vmid, $cpustr);
4585
4586             my $retry = 0;
4587             my $currentrunningvcpus = undef;
4588             while (1) {
4589                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4590                 last if scalar(@{$currentrunningvcpus}) == $i;
4591                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4592                 sleep 1;
4593                 $retry++;
4594             }
4595             #update conf after each succesfull cpu hotplug
4596             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4597             PVE::QemuConfig->write_config($vmid, $conf);
4598         }
4599     } else {
4600
4601         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4602             mon_cmd($vmid, "cpu-add", id => int($i));
4603         }
4604     }
4605 }
4606
4607 sub qemu_block_set_io_throttle {
4608     my ($vmid, $deviceid,
4609         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4610         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4611         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4612         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4613
4614     return if !check_running($vmid) ;
4615
4616     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4617         bps => int($bps),
4618         bps_rd => int($bps_rd),
4619         bps_wr => int($bps_wr),
4620         iops => int($iops),
4621         iops_rd => int($iops_rd),
4622         iops_wr => int($iops_wr),
4623         bps_max => int($bps_max),
4624         bps_rd_max => int($bps_rd_max),
4625         bps_wr_max => int($bps_wr_max),
4626         iops_max => int($iops_max),
4627         iops_rd_max => int($iops_rd_max),
4628         iops_wr_max => int($iops_wr_max),
4629         bps_max_length => int($bps_max_length),
4630         bps_rd_max_length => int($bps_rd_max_length),
4631         bps_wr_max_length => int($bps_wr_max_length),
4632         iops_max_length => int($iops_max_length),
4633         iops_rd_max_length => int($iops_rd_max_length),
4634         iops_wr_max_length => int($iops_wr_max_length),
4635     );
4636
4637 }
4638
4639 sub qemu_block_resize {
4640     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4641
4642     my $running = check_running($vmid);
4643
4644     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4645
4646     return if !$running;
4647
4648     my $padding = (1024 - $size % 1024) % 1024;
4649     $size = $size + $padding;
4650
4651     mon_cmd(
4652         $vmid,
4653         "block_resize",
4654         device => $deviceid,
4655         size => int($size),
4656         timeout => 60,
4657     );
4658 }
4659
4660 sub qemu_volume_snapshot {
4661     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4662
4663     my $running = check_running($vmid);
4664
4665     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4666         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4667     } else {
4668         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4669     }
4670 }
4671
4672 sub qemu_volume_snapshot_delete {
4673     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4674
4675     my $running = check_running($vmid);
4676     my $attached_deviceid;
4677
4678     if ($running) {
4679         my $conf = PVE::QemuConfig->load_config($vmid);
4680         PVE::QemuConfig->foreach_volume($conf, sub {
4681             my ($ds, $drive) = @_;
4682             $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4683         });
4684     }
4685
4686     if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4687         mon_cmd(
4688             $vmid,
4689             'blockdev-snapshot-delete-internal-sync',
4690             device => $attached_deviceid,
4691             name => $snap,
4692         );
4693     } else {
4694         PVE::Storage::volume_snapshot_delete(
4695             $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4696     }
4697 }
4698
4699 sub set_migration_caps {
4700     my ($vmid, $savevm) = @_;
4701
4702     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4703
4704     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4705     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4706
4707     my $cap_ref = [];
4708
4709     my $enabled_cap = {
4710         "auto-converge" => 1,
4711         "xbzrle" => 1,
4712         "x-rdma-pin-all" => 0,
4713         "zero-blocks" => 0,
4714         "compress" => 0,
4715         "dirty-bitmaps" => $dirty_bitmaps,
4716     };
4717
4718     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4719
4720     for my $supported_capability (@$supported_capabilities) {
4721         push @$cap_ref, {
4722             capability => $supported_capability->{capability},
4723             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4724         };
4725     }
4726
4727     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4728 }
4729
4730 sub foreach_volid {
4731     my ($conf, $func, @param) = @_;
4732
4733     my $volhash = {};
4734
4735     my $test_volid = sub {
4736         my ($key, $drive, $snapname, $pending) = @_;
4737
4738         my $volid = $drive->{file};
4739         return if !$volid;
4740
4741         $volhash->{$volid}->{cdrom} //= 1;
4742         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4743
4744         my $replicate = $drive->{replicate} // 1;
4745         $volhash->{$volid}->{replicate} //= 0;
4746         $volhash->{$volid}->{replicate} = 1 if $replicate;
4747
4748         $volhash->{$volid}->{shared} //= 0;
4749         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4750
4751         $volhash->{$volid}->{is_unused} //= 0;
4752         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4753
4754         $volhash->{$volid}->{is_attached} //= 0;
4755         $volhash->{$volid}->{is_attached} = 1
4756             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4757
4758         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4759             if defined($snapname);
4760
4761         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4762
4763         my $size = $drive->{size};
4764         $volhash->{$volid}->{size} //= $size if $size;
4765
4766         $volhash->{$volid}->{is_vmstate} //= 0;
4767         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4768
4769         $volhash->{$volid}->{is_tpmstate} //= 0;
4770         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4771
4772         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4773     };
4774
4775     my $include_opts = {
4776         extra_keys => ['vmstate'],
4777         include_unused => 1,
4778     };
4779
4780     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4781
4782     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4783         if defined($conf->{pending}) && $conf->{pending}->%*;
4784
4785     foreach my $snapname (keys %{$conf->{snapshots}}) {
4786         my $snap = $conf->{snapshots}->{$snapname};
4787         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4788     }
4789
4790     foreach my $volid (keys %$volhash) {
4791         &$func($volid, $volhash->{$volid}, @param);
4792     }
4793 }
4794
4795 my $fast_plug_option = {
4796     'description' => 1,
4797     'hookscript' => 1,
4798     'lock' => 1,
4799     'migrate_downtime' => 1,
4800     'migrate_speed' => 1,
4801     'name' => 1,
4802     'onboot' => 1,
4803     'protection' => 1,
4804     'shares' => 1,
4805     'startup' => 1,
4806     'tags' => 1,
4807     'vmstatestorage' => 1,
4808 };
4809
4810 for my $opt (keys %$confdesc_cloudinit) {
4811     $fast_plug_option->{$opt} = 1;
4812 };
4813
4814 # hotplug changes in [PENDING]
4815 # $selection hash can be used to only apply specified options, for
4816 # example: { cores => 1 } (only apply changed 'cores')
4817 # $errors ref is used to return error messages
4818 sub vmconfig_hotplug_pending {
4819     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4820
4821     my $defaults = load_defaults();
4822     my $arch = get_vm_arch($conf);
4823     my $machine_type = get_vm_machine($conf, undef, $arch);
4824
4825     # commit values which do not have any impact on running VM first
4826     # Note: those option cannot raise errors, we we do not care about
4827     # $selection and always apply them.
4828
4829     my $add_error = sub {
4830         my ($opt, $msg) = @_;
4831         $errors->{$opt} = "hotplug problem - $msg";
4832     };
4833
4834     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4835
4836     my $cloudinit_record_changed = sub {
4837         my ($conf, $opt, $old, $new) = @_;
4838         return if !$cloudinit_pending_properties->{$opt};
4839
4840         my $ci = ($conf->{cloudinit} //= {});
4841
4842         my $recorded = $ci->{$opt};
4843         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4844
4845         if (defined($new)) {
4846             if (defined($old)) {
4847                 # an existing value is being modified
4848                 if (defined($recorded)) {
4849                     # the value was already not in sync
4850                     if ($new eq $recorded) {
4851                         # a value is being reverted to the cloud-init state:
4852                         delete $ci->{$opt};
4853                         delete $added{$opt};
4854                     } else {
4855                         # the value was changed multiple times, do nothing
4856                     }
4857                 } elsif ($added{$opt}) {
4858                     # the value had been marked as added and is being changed, do nothing
4859                 } else {
4860                     # the value is new, record it:
4861                     $ci->{$opt} = $old;
4862                 }
4863             } else {
4864                 # a new value is being added
4865                 if (defined($recorded)) {
4866                     # it was already not in sync
4867                     if ($new eq $recorded) {
4868                         # a value is being reverted to the cloud-init state:
4869                         delete $ci->{$opt};
4870                         delete $added{$opt};
4871                     } else {
4872                         # the value had temporarily been removed, do nothing
4873                     }
4874                 } elsif ($added{$opt}) {
4875                     # the value had been marked as added already, do nothing
4876                 } else {
4877                     # the value is new, add it
4878                     $added{$opt} = 1;
4879                 }
4880             }
4881         } elsif (!defined($old)) {
4882             # a non-existent value is being removed? ignore...
4883         } else {
4884             # a value is being deleted
4885             if (defined($recorded)) {
4886                 # a value was already recorded, just keep it
4887             } elsif ($added{$opt}) {
4888                 # the value was marked as added, remove it
4889                 delete $added{$opt};
4890             } else {
4891                 # a previously unrecorded value is being removed, record the old value:
4892                 $ci->{$opt} = $old;
4893             }
4894         }
4895
4896         my $added = join(',', sort keys %added);
4897         $ci->{added} = $added if length($added);
4898     };
4899
4900     my $changes = 0;
4901     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4902         if ($fast_plug_option->{$opt}) {
4903             my $new = delete $conf->{pending}->{$opt};
4904             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4905             $conf->{$opt} = $new;
4906             $changes = 1;
4907         }
4908     }
4909
4910     if ($changes) {
4911         PVE::QemuConfig->write_config($vmid, $conf);
4912     }
4913
4914     my $ostype = $conf->{ostype};
4915     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4916     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4917     my $usb_hotplug = $hotplug_features->{usb}
4918         && min_version($version, 7, 1)
4919         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4920
4921     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4922     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4923
4924     foreach my $opt (sort keys %$pending_delete_hash) {
4925         next if $selection && !$selection->{$opt};
4926         my $force = $pending_delete_hash->{$opt}->{force};
4927         eval {
4928             if ($opt eq 'hotplug') {
4929                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4930             } elsif ($opt eq 'tablet') {
4931                 die "skip\n" if !$hotplug_features->{usb};
4932                 if ($defaults->{tablet}) {
4933                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4934                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4935                         if $arch eq 'aarch64';
4936                 } else {
4937                     vm_deviceunplug($vmid, $conf, 'tablet');
4938                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4939                 }
4940             } elsif ($opt =~ m/^usb(\d+)$/) {
4941                 my $index = $1;
4942                 die "skip\n" if !$usb_hotplug;
4943                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4944                 vm_deviceunplug($vmid, $conf, $opt);
4945             } elsif ($opt eq 'vcpus') {
4946                 die "skip\n" if !$hotplug_features->{cpu};
4947                 qemu_cpu_hotplug($vmid, $conf, undef);
4948             } elsif ($opt eq 'balloon') {
4949                 # enable balloon device is not hotpluggable
4950                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4951                 # here we reset the ballooning value to memory
4952                 my $balloon = get_current_memory($conf->{memory});
4953                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4954             } elsif ($fast_plug_option->{$opt}) {
4955                 # do nothing
4956             } elsif ($opt =~ m/^net(\d+)$/) {
4957                 die "skip\n" if !$hotplug_features->{network};
4958                 vm_deviceunplug($vmid, $conf, $opt);
4959                 if($have_sdn) {
4960                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
4961                     PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
4962                 }
4963             } elsif (is_valid_drivename($opt)) {
4964                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4965                 vm_deviceunplug($vmid, $conf, $opt);
4966                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4967             } elsif ($opt =~ m/^memory$/) {
4968                 die "skip\n" if !$hotplug_features->{memory};
4969                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
4970             } elsif ($opt eq 'cpuunits') {
4971                 $cgroup->change_cpu_shares(undef);
4972             } elsif ($opt eq 'cpulimit') {
4973                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4974             } else {
4975                 die "skip\n";
4976             }
4977         };
4978         if (my $err = $@) {
4979             &$add_error($opt, $err) if $err ne "skip\n";
4980         } else {
4981             my $old = delete $conf->{$opt};
4982             $cloudinit_record_changed->($conf, $opt, $old, undef);
4983             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4984         }
4985     }
4986
4987     my $cloudinit_opt;
4988     foreach my $opt (keys %{$conf->{pending}}) {
4989         next if $selection && !$selection->{$opt};
4990         my $value = $conf->{pending}->{$opt};
4991         eval {
4992             if ($opt eq 'hotplug') {
4993                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4994                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
4995             } elsif ($opt eq 'tablet') {
4996                 die "skip\n" if !$hotplug_features->{usb};
4997                 if ($value == 1) {
4998                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4999                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5000                         if $arch eq 'aarch64';
5001                 } elsif ($value == 0) {
5002                     vm_deviceunplug($vmid, $conf, 'tablet');
5003                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5004                 }
5005             } elsif ($opt =~ m/^usb(\d+)$/) {
5006                 my $index = $1;
5007                 die "skip\n" if !$usb_hotplug;
5008                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5009                 my $id = $opt;
5010                 if ($d->{host} =~ m/^spice$/i)  {
5011                     $id = "usbredirdev$index";
5012                 }
5013                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5014             } elsif ($opt eq 'vcpus') {
5015                 die "skip\n" if !$hotplug_features->{cpu};
5016                 qemu_cpu_hotplug($vmid, $conf, $value);
5017             } elsif ($opt eq 'balloon') {
5018                 # enable/disable balloning device is not hotpluggable
5019                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5020                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5021                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5022
5023                 # allow manual ballooning if shares is set to zero
5024                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5025                     my $memory = get_current_memory($conf->{memory});
5026                     my $balloon = $conf->{pending}->{balloon} || $memory;
5027                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5028                 }
5029             } elsif ($opt =~ m/^net(\d+)$/) {
5030                 # some changes can be done without hotplug
5031                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5032                                     $vmid, $opt, $value, $arch, $machine_type);
5033             } elsif (is_valid_drivename($opt)) {
5034                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5035                 # some changes can be done without hotplug
5036                 my $drive = parse_drive($opt, $value);
5037                 if (drive_is_cloudinit($drive)) {
5038                     $cloudinit_opt = [$opt, $drive];
5039                     # apply all the other changes first, then generate the cloudinit disk
5040                     die "skip\n";
5041                 }
5042                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5043                                      $vmid, $opt, $value, $arch, $machine_type);
5044             } elsif ($opt =~ m/^memory$/) { #dimms
5045                 die "skip\n" if !$hotplug_features->{memory};
5046                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5047             } elsif ($opt eq 'cpuunits') {
5048                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5049                 $cgroup->change_cpu_shares($new_cpuunits);
5050             } elsif ($opt eq 'cpulimit') {
5051                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5052                 $cgroup->change_cpu_quota($cpulimit, 100000);
5053             } elsif ($opt eq 'agent') {
5054                 vmconfig_update_agent($conf, $opt, $value);
5055             } else {
5056                 die "skip\n";  # skip non-hot-pluggable options
5057             }
5058         };
5059         if (my $err = $@) {
5060             &$add_error($opt, $err) if $err ne "skip\n";
5061         } else {
5062             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5063             $conf->{$opt} = $value;
5064             delete $conf->{pending}->{$opt};
5065         }
5066     }
5067
5068     if (defined($cloudinit_opt)) {
5069         my ($opt, $drive) = @$cloudinit_opt;
5070         my $value = $conf->{pending}->{$opt};
5071         eval {
5072             my $temp = {%$conf, $opt => $value};
5073             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5074             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5075                                  $vmid, $opt, $value, $arch, $machine_type);
5076         };
5077         if (my $err = $@) {
5078             &$add_error($opt, $err) if $err ne "skip\n";
5079         } else {
5080             $conf->{$opt} = $value;
5081             delete $conf->{pending}->{$opt};
5082         }
5083     }
5084
5085     # unplug xhci controller if no usb device is left
5086     if ($usb_hotplug) {
5087         my $has_usb = 0;
5088         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5089             next if !defined($conf->{"usb$i"});
5090             $has_usb = 1;
5091             last;
5092         }
5093         if (!$has_usb) {
5094             vm_deviceunplug($vmid, $conf, 'xhci');
5095         }
5096     }
5097
5098     PVE::QemuConfig->write_config($vmid, $conf);
5099
5100     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5101         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5102     }
5103 }
5104
5105 sub try_deallocate_drive {
5106     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5107
5108     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5109         my $volid = $drive->{file};
5110         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5111             my $sid = PVE::Storage::parse_volume_id($volid);
5112             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5113
5114             # check if the disk is really unused
5115             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5116                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5117             PVE::Storage::vdisk_free($storecfg, $volid);
5118             return 1;
5119         } else {
5120             # If vm is not owner of this disk remove from config
5121             return 1;
5122         }
5123     }
5124
5125     return;
5126 }
5127
5128 sub vmconfig_delete_or_detach_drive {
5129     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5130
5131     my $drive = parse_drive($opt, $conf->{$opt});
5132
5133     my $rpcenv = PVE::RPCEnvironment::get();
5134     my $authuser = $rpcenv->get_user();
5135
5136     if ($force) {
5137         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5138         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5139     } else {
5140         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5141     }
5142 }
5143
5144
5145
5146 sub vmconfig_apply_pending {
5147     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5148
5149     return if !scalar(keys %{$conf->{pending}});
5150
5151     my $add_apply_error = sub {
5152         my ($opt, $msg) = @_;
5153         my $err_msg = "unable to apply pending change $opt : $msg";
5154         $errors->{$opt} = $err_msg;
5155         warn $err_msg;
5156     };
5157
5158     # cold plug
5159
5160     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5161     foreach my $opt (sort keys %$pending_delete_hash) {
5162         my $force = $pending_delete_hash->{$opt}->{force};
5163         eval {
5164             if ($opt =~ m/^unused/) {
5165                 die "internal error";
5166             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5167                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5168             } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5169                 if($have_sdn) {
5170                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5171                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5172                     warn if $@;
5173                 }
5174             }
5175         };
5176         if (my $err = $@) {
5177             $add_apply_error->($opt, $err);
5178         } else {
5179             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5180             delete $conf->{$opt};
5181         }
5182     }
5183
5184     PVE::QemuConfig->cleanup_pending($conf);
5185
5186     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5187
5188     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5189         next if $opt eq 'delete'; # just to be sure
5190         eval {
5191             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5192                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5193             } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5194                 if($have_sdn) {
5195                     my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5196                     if ($conf->{$opt}){
5197                         my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5198
5199                         if ($old_net->{bridge} ne $new_net->{bridge} ||
5200                             $old_net->{macaddr} ne $new_net->{macaddr}) {
5201                             PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5202                         }
5203                    }
5204                    #fixme: reuse ip if mac change && same bridge
5205                    PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5206                 }
5207             }
5208         };
5209         if (my $err = $@) {
5210             $add_apply_error->($opt, $err);
5211         } else {
5212
5213             if (is_valid_drivename($opt)) {
5214                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5215                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5216             }
5217
5218             $conf->{$opt} = delete $conf->{pending}->{$opt};
5219         }
5220     }
5221
5222     # write all changes at once to avoid unnecessary i/o
5223     PVE::QemuConfig->write_config($vmid, $conf);
5224     if ($generate_cloudinit) {
5225         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5226             # After successful generation and if there were changes to be applied, update the
5227             # config to drop the {cloudinit} entry.
5228             PVE::QemuConfig->write_config($vmid, $conf);
5229         }
5230     }
5231 }
5232
5233 sub vmconfig_update_net {
5234     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5235
5236     my $newnet = parse_net($value);
5237
5238     if ($conf->{$opt}) {
5239         my $oldnet = parse_net($conf->{$opt});
5240
5241         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5242             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5243             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5244             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5245             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5246
5247             # for non online change, we try to hot-unplug
5248             die "skip\n" if !$hotplug;
5249             vm_deviceunplug($vmid, $conf, $opt);
5250
5251             if($have_sdn) {
5252                 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5253             }
5254
5255         } else {
5256
5257             die "internal error" if $opt !~ m/net(\d+)/;
5258             my $iface = "tap${vmid}i$1";
5259
5260             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5261                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5262                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5263                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5264                 PVE::Network::tap_unplug($iface);
5265
5266                 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5267                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5268                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5269                     qemu_set_link_status($vmid, $opt, 0);
5270                 }
5271
5272                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5273                     if ($have_sdn) {
5274                         PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5275                         PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5276                     }
5277                 }
5278
5279                 if ($have_sdn) {
5280                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5281                 } else {
5282                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5283                 }
5284
5285                 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5286                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5287                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5288                     qemu_set_link_status($vmid, $opt, 1);
5289                 }
5290
5291             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5292                 # Rate can be applied on its own but any change above needs to
5293                 # include the rate in tap_plug since OVS resets everything.
5294                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5295             }
5296
5297             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5298                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5299             }
5300
5301             return 1;
5302         }
5303     }
5304
5305     if ($hotplug) {
5306         if ($have_sdn) {
5307             PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5308             PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5309         }
5310         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5311     } else {
5312         die "skip\n";
5313     }
5314 }
5315
5316 sub vmconfig_update_agent {
5317     my ($conf, $opt, $value) = @_;
5318
5319     die "skip\n" if !$conf->{$opt};
5320
5321     my $hotplug_options = { fstrim_cloned_disks => 1 };
5322
5323     my $old_agent = parse_guest_agent($conf);
5324     my $agent = parse_guest_agent({$opt => $value});
5325
5326     for my $option (keys %$agent) { # added/changed options
5327         next if defined($hotplug_options->{$option});
5328         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5329     }
5330
5331     for my $option (keys %$old_agent) { # removed options
5332         next if defined($hotplug_options->{$option});
5333         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5334     }
5335
5336     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5337 }
5338
5339 sub vmconfig_update_disk {
5340     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5341
5342     my $drive = parse_drive($opt, $value);
5343
5344     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5345         my $media = $drive->{media} || 'disk';
5346         my $oldmedia = $old_drive->{media} || 'disk';
5347         die "unable to change media type\n" if $media ne $oldmedia;
5348
5349         if (!drive_is_cdrom($old_drive)) {
5350
5351             if ($drive->{file} ne $old_drive->{file}) {
5352
5353                 die "skip\n" if !$hotplug;
5354
5355                 # unplug and register as unused
5356                 vm_deviceunplug($vmid, $conf, $opt);
5357                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5358
5359             } else {
5360                 # update existing disk
5361
5362                 # skip non hotpluggable value
5363                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5364                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5365                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5366                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5367                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5368                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5369                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5370                     die "skip\n";
5371                 }
5372
5373                 # apply throttle
5374                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5375                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5376                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5377                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5378                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5379                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5380                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5381                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5382                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5383                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5384                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5385                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5386                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5387                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5388                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5389                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5390                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5391                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5392
5393                     qemu_block_set_io_throttle(
5394                         $vmid,"drive-$opt",
5395                         ($drive->{mbps} || 0)*1024*1024,
5396                         ($drive->{mbps_rd} || 0)*1024*1024,
5397                         ($drive->{mbps_wr} || 0)*1024*1024,
5398                          $drive->{iops} || 0,
5399                          $drive->{iops_rd} || 0,
5400                          $drive->{iops_wr} || 0,
5401                         ($drive->{mbps_max} || 0)*1024*1024,
5402                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5403                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5404                          $drive->{iops_max} || 0,
5405                          $drive->{iops_rd_max} || 0,
5406                          $drive->{iops_wr_max} || 0,
5407                          $drive->{bps_max_length} || 1,
5408                          $drive->{bps_rd_max_length} || 1,
5409                          $drive->{bps_wr_max_length} || 1,
5410                          $drive->{iops_max_length} || 1,
5411                          $drive->{iops_rd_max_length} || 1,
5412                          $drive->{iops_wr_max_length} || 1,
5413                     );
5414
5415                 }
5416
5417                 return 1;
5418             }
5419
5420         } else { # cdrom
5421
5422             if ($drive->{file} eq 'none') {
5423                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5424                 if (drive_is_cloudinit($old_drive)) {
5425                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5426                 }
5427             } else {
5428                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5429
5430                 # force eject if locked
5431                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5432
5433                 if ($path) {
5434                     mon_cmd($vmid, "blockdev-change-medium",
5435                         id => "$opt", filename => "$path");
5436                 }
5437             }
5438
5439             return 1;
5440         }
5441     }
5442
5443     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5444     # hotplug new disks
5445     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5446     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5447 }
5448
5449 sub vmconfig_update_cloudinit_drive {
5450     my ($storecfg, $conf, $vmid) = @_;
5451
5452     my $cloudinit_ds = undef;
5453     my $cloudinit_drive = undef;
5454
5455     PVE::QemuConfig->foreach_volume($conf, sub {
5456         my ($ds, $drive) = @_;
5457         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5458             $cloudinit_ds = $ds;
5459             $cloudinit_drive = $drive;
5460         }
5461     });
5462
5463     return if !$cloudinit_drive;
5464
5465     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5466         PVE::QemuConfig->write_config($vmid, $conf);
5467     }
5468
5469     my $running = PVE::QemuServer::check_running($vmid);
5470
5471     if ($running) {
5472         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5473         if ($path) {
5474             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5475             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5476         }
5477     }
5478 }
5479
5480 # called in locked context by incoming migration
5481 sub vm_migrate_get_nbd_disks {
5482     my ($storecfg, $conf, $replicated_volumes) = @_;
5483
5484     my $local_volumes = {};
5485     PVE::QemuConfig->foreach_volume($conf, sub {
5486         my ($ds, $drive) = @_;
5487
5488         return if drive_is_cdrom($drive);
5489         return if $ds eq 'tpmstate0';
5490
5491         my $volid = $drive->{file};
5492
5493         return if !$volid;
5494
5495         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5496
5497         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5498         return if $scfg->{shared};
5499
5500         my $format = qemu_img_format($scfg, $volname);
5501
5502         # replicated disks re-use existing state via bitmap
5503         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5504         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5505     });
5506     return $local_volumes;
5507 }
5508
5509 # called in locked context by incoming migration
5510 sub vm_migrate_alloc_nbd_disks {
5511     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5512
5513     my $nbd = {};
5514     foreach my $opt (sort keys %$source_volumes) {
5515         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5516
5517         if ($use_existing) {
5518             $nbd->{$opt}->{drivestr} = print_drive($drive);
5519             $nbd->{$opt}->{volid} = $volid;
5520             $nbd->{$opt}->{replicated} = 1;
5521             next;
5522         }
5523
5524         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5525
5526         # order of precedence, filtered by whether storage supports it:
5527         # 1. explicit requested format
5528         # 2. default format of storage
5529         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5530         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5531
5532         my $size = $drive->{size} / 1024;
5533         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5534         my $newdrive = $drive;
5535         $newdrive->{format} = $format;
5536         $newdrive->{file} = $newvolid;
5537         my $drivestr = print_drive($newdrive);
5538         $nbd->{$opt}->{drivestr} = $drivestr;
5539         $nbd->{$opt}->{volid} = $newvolid;
5540     }
5541
5542     return $nbd;
5543 }
5544
5545 # see vm_start_nolock for parameters, additionally:
5546 # migrate_opts:
5547 #   storagemap = parsed storage map for allocating NBD disks
5548 sub vm_start {
5549     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5550
5551     return PVE::QemuConfig->lock_config($vmid, sub {
5552         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5553
5554         die "you can't start a vm if it's a template\n"
5555             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5556
5557         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5558         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5559
5560         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5561
5562         if ($has_backup_lock && $running) {
5563             # a backup is currently running, attempt to start the guest in the
5564             # existing QEMU instance
5565             return vm_resume($vmid);
5566         }
5567
5568         PVE::QemuConfig->check_lock($conf)
5569             if !($params->{skiplock} || $has_suspended_lock);
5570
5571         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5572
5573         die "VM $vmid already running\n" if $running;
5574
5575         if (my $storagemap = $migrate_opts->{storagemap}) {
5576             my $replicated = $migrate_opts->{replicated_volumes};
5577             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5578             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5579
5580             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5581                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5582             }
5583         }
5584
5585         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5586     });
5587 }
5588
5589
5590 # params:
5591 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5592 #   skiplock => 0/1, skip checking for config lock
5593 #   skiptemplate => 0/1, skip checking whether VM is template
5594 #   forcemachine => to force QEMU machine (rollback/migration)
5595 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5596 #   timeout => in seconds
5597 #   paused => start VM in paused state (backup)
5598 #   resume => resume from hibernation
5599 #   pbs-backing => {
5600 #      sata0 => {
5601 #         repository
5602 #         snapshot
5603 #         keyfile
5604 #         archive
5605 #      },
5606 #      virtio2 => ...
5607 #   }
5608 # migrate_opts:
5609 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5610 #   migratedfrom => source node
5611 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5612 #   network => CIDR of migration network
5613 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5614 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5615 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5616 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5617 #       contained in config
5618 sub vm_start_nolock {
5619     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5620
5621     my $statefile = $params->{statefile};
5622     my $resume = $params->{resume};
5623
5624     my $migratedfrom = $migrate_opts->{migratedfrom};
5625     my $migration_type = $migrate_opts->{type};
5626
5627     my $res = {};
5628
5629     # clean up leftover reboot request files
5630     eval { clear_reboot_request($vmid); };
5631     warn $@ if $@;
5632
5633     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5634         vmconfig_apply_pending($vmid, $conf, $storecfg);
5635         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5636     }
5637
5638     # don't regenerate the ISO if the VM is started as part of a live migration
5639     # this way we can reuse the old ISO with the correct config
5640     if (!$migratedfrom) {
5641         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5642             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5643             # $conf->{cloudinit}, so we could just not do this?
5644             # But we do it above, so for now let's be consistent.
5645             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5646         }
5647     }
5648
5649     # override offline migrated volumes, conf is out of date still
5650     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5651         for my $key (sort keys $offline_volumes->%*) {
5652             my $parsed = parse_drive($key, $conf->{$key});
5653             $parsed->{file} = $offline_volumes->{$key};
5654             $conf->{$key} = print_drive($parsed);
5655         }
5656     }
5657
5658     my $defaults = load_defaults();
5659
5660     # set environment variable useful inside network script
5661     # for remote migration the config is available on the target node!
5662     if (!$migrate_opts->{remote_node}) {
5663         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5664     }
5665
5666     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5667
5668     my $forcemachine = $params->{forcemachine};
5669     my $forcecpu = $params->{forcecpu};
5670     if ($resume) {
5671         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5672         $forcemachine = $conf->{runningmachine};
5673         $forcecpu = $conf->{runningcpu};
5674         print "Resuming suspended VM\n";
5675     }
5676
5677     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5678         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5679
5680     my $migration_ip;
5681     my $get_migration_ip = sub {
5682         my ($nodename) = @_;
5683
5684         return $migration_ip if defined($migration_ip);
5685
5686         my $cidr = $migrate_opts->{network};
5687
5688         if (!defined($cidr)) {
5689             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5690             $cidr = $dc_conf->{migration}->{network};
5691         }
5692
5693         if (defined($cidr)) {
5694             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5695
5696             die "could not get IP: no address configured on local " .
5697                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5698
5699             die "could not get IP: multiple addresses configured on local " .
5700                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5701
5702             $migration_ip = @$ips[0];
5703         }
5704
5705         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5706             if !defined($migration_ip);
5707
5708         return $migration_ip;
5709     };
5710
5711     if ($statefile) {
5712         if ($statefile eq 'tcp') {
5713             my $migrate = $res->{migrate} = { proto => 'tcp' };
5714             $migrate->{addr} = "localhost";
5715             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5716             my $nodename = nodename();
5717
5718             if (!defined($migration_type)) {
5719                 if (defined($datacenterconf->{migration}->{type})) {
5720                     $migration_type = $datacenterconf->{migration}->{type};
5721                 } else {
5722                     $migration_type = 'secure';
5723                 }
5724             }
5725
5726             if ($migration_type eq 'insecure') {
5727                 $migrate->{addr} = $get_migration_ip->($nodename);
5728                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5729             }
5730
5731             # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5732             # via QMP later
5733             push @$cmd, '-incoming', 'defer';
5734             push @$cmd, '-S';
5735
5736         } elsif ($statefile eq 'unix') {
5737             # should be default for secure migrations as a ssh TCP forward
5738             # tunnel is not deterministic reliable ready and fails regurarly
5739             # to set up in time, so use UNIX socket forwards
5740             my $migrate = $res->{migrate} = { proto => 'unix' };
5741             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5742             unlink $migrate->{addr};
5743
5744             $migrate->{uri} = "unix:$migrate->{addr}";
5745             push @$cmd, '-incoming', $migrate->{uri};
5746             push @$cmd, '-S';
5747
5748         } elsif (-e $statefile) {
5749             push @$cmd, '-loadstate', $statefile;
5750         } else {
5751             my $statepath = PVE::Storage::path($storecfg, $statefile);
5752             push @$vollist, $statefile;
5753             push @$cmd, '-loadstate', $statepath;
5754         }
5755     } elsif ($params->{paused}) {
5756         push @$cmd, '-S';
5757     }
5758
5759     my $memory = get_current_memory($conf->{memory});
5760     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5761
5762     my $pci_reserve_list = [];
5763     for my $device (values $pci_devices->%*) {
5764         next if $device->{mdev}; # we don't reserve for mdev devices
5765         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5766     }
5767
5768     # reserve all PCI IDs before actually doing anything with them
5769     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5770
5771     eval {
5772         my $uuid;
5773         for my $id (sort keys %$pci_devices) {
5774             my $d = $pci_devices->{$id};
5775             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5776
5777             my $chosen_mdev;
5778             for my $dev ($d->{ids}->@*) {
5779                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5780                 if ($d->{mdev}) {
5781                     warn $@ if $@;
5782                     $chosen_mdev = $info;
5783                     last if $chosen_mdev; # if successful, we're done
5784                 } else {
5785                     die $@ if $@;
5786                 }
5787             }
5788
5789             next if !$d->{mdev};
5790             die "could not create mediated device\n" if !defined($chosen_mdev);
5791
5792             # nvidia grid needs the uuid of the mdev as qemu parameter
5793             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5794                 if (defined($conf->{smbios1})) {
5795                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5796                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5797                 }
5798                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5799             }
5800         }
5801         push @$cmd, '-uuid', $uuid if defined($uuid);
5802     };
5803     if (my $err = $@) {
5804         eval { cleanup_pci_devices($vmid, $conf) };
5805         warn $@ if $@;
5806         die $err;
5807     }
5808
5809     PVE::Storage::activate_volumes($storecfg, $vollist);
5810
5811
5812     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5813     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5814     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5815     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5816     # timeout should be more than enough here...
5817     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5818
5819     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5820
5821     my %run_params = (
5822         timeout => $statefile ? undef : $start_timeout,
5823         umask => 0077,
5824         noerr => 1,
5825     );
5826
5827     # when migrating, prefix QEMU output so other side can pick up any
5828     # errors that might occur and show the user
5829     if ($migratedfrom) {
5830         $run_params{quiet} = 1;
5831         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5832     }
5833
5834     my %systemd_properties = (
5835         Slice => 'qemu.slice',
5836         KillMode => 'process',
5837         SendSIGKILL => 0,
5838         TimeoutStopUSec => ULONG_MAX, # infinity
5839     );
5840
5841     if (PVE::CGroup::cgroup_mode() == 2) {
5842         $systemd_properties{CPUWeight} = $cpuunits;
5843     } else {
5844         $systemd_properties{CPUShares} = $cpuunits;
5845     }
5846
5847     if (my $cpulimit = $conf->{cpulimit}) {
5848         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5849     }
5850     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5851
5852     my $run_qemu = sub {
5853         PVE::Tools::run_fork sub {
5854             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5855
5856             my $tpmpid;
5857             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5858                 # start the TPM emulator so QEMU can connect on start
5859                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5860             }
5861
5862             my $exitcode = run_command($cmd, %run_params);
5863             if ($exitcode) {
5864                 if ($tpmpid) {
5865                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5866                     kill 'TERM', $tpmpid;
5867                 }
5868                 die "QEMU exited with code $exitcode\n";
5869             }
5870         };
5871     };
5872
5873     if ($conf->{hugepages}) {
5874
5875         my $code = sub {
5876             my $hotplug_features =
5877                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5878             my $hugepages_topology =
5879                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5880
5881             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5882
5883             PVE::QemuServer::Memory::hugepages_mount();
5884             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5885
5886             eval { $run_qemu->() };
5887             if (my $err = $@) {
5888                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5889                     if !$conf->{keephugepages};
5890                 die $err;
5891             }
5892
5893             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5894                 if !$conf->{keephugepages};
5895         };
5896         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5897
5898     } else {
5899         eval { $run_qemu->() };
5900     }
5901
5902     if (my $err = $@) {
5903         # deactivate volumes if start fails
5904         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5905         warn $@ if $@;
5906         eval { cleanup_pci_devices($vmid, $conf) };
5907         warn $@ if $@;
5908
5909         die "start failed: $err";
5910     }
5911
5912     # re-reserve all PCI IDs now that we can know the actual VM PID
5913     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5914     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5915     warn $@ if $@;
5916
5917     if (defined(my $migrate = $res->{migrate})) {
5918         if ($migrate->{proto} eq 'tcp') {
5919             my $nodename = nodename();
5920             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5921             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5922             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5923             mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
5924         }
5925         print "migration listens on $migrate->{uri}\n";
5926     } elsif ($statefile) {
5927         eval { mon_cmd($vmid, "cont"); };
5928         warn $@ if $@;
5929     }
5930
5931     #start nbd server for storage migration
5932     if (my $nbd = $migrate_opts->{nbd}) {
5933         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5934
5935         my $migrate_storage_uri;
5936         # nbd_protocol_version > 0 for unix socket support
5937         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5938             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5939             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5940             $migrate_storage_uri = "nbd:unix:$socket_path";
5941             $res->{migrate}->{unix_sockets} = [$socket_path];
5942         } else {
5943             my $nodename = nodename();
5944             my $localip = $get_migration_ip->($nodename);
5945             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5946             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5947
5948             mon_cmd($vmid, "nbd-server-start", addr => {
5949                 type => 'inet',
5950                 data => {
5951                     host => "${localip}",
5952                     port => "${storage_migrate_port}",
5953                 },
5954             });
5955             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5956             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5957         }
5958
5959         my $block_info = mon_cmd($vmid, "query-block");
5960         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5961
5962         foreach my $opt (sort keys %$nbd) {
5963             my $drivestr = $nbd->{$opt}->{drivestr};
5964             my $volid = $nbd->{$opt}->{volid};
5965
5966             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5967
5968             mon_cmd(
5969                 $vmid,
5970                 "block-export-add",
5971                 id => "drive-$opt",
5972                 'node-name' => $block_node,
5973                 writable => JSON::true,
5974                 type => "nbd",
5975                 name => "drive-$opt", # NBD export name
5976             );
5977
5978             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5979             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5980             print "re-using replicated volume: $opt - $volid\n"
5981                 if $nbd->{$opt}->{replicated};
5982
5983             $res->{drives}->{$opt} = $nbd->{$opt};
5984             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5985         }
5986     }
5987
5988     if ($migratedfrom) {
5989         eval {
5990             set_migration_caps($vmid);
5991         };
5992         warn $@ if $@;
5993
5994         if ($spice_port) {
5995             print "spice listens on port $spice_port\n";
5996             $res->{spice_port} = $spice_port;
5997             if ($migrate_opts->{spice_ticket}) {
5998                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5999                     $migrate_opts->{spice_ticket});
6000                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6001             }
6002         }
6003
6004     } else {
6005         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6006             if !$statefile && $conf->{balloon};
6007
6008         foreach my $opt (keys %$conf) {
6009             next if $opt !~  m/^net\d+$/;
6010             my $nicconf = parse_net($conf->{$opt});
6011             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6012         }
6013         add_nets_bridge_fdb($conf, $vmid);
6014     }
6015
6016    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6017         eval {
6018             mon_cmd(
6019                 $vmid,
6020                 'qom-set',
6021                 path => "machine/peripheral/balloon0",
6022                 property => "guest-stats-polling-interval",
6023                 value => 2
6024             );
6025         };
6026         log_warn("could not set polling interval for ballooning - $@") if $@;
6027     }
6028
6029     if ($resume) {
6030         print "Resumed VM, removing state\n";
6031         if (my $vmstate = $conf->{vmstate}) {
6032             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6033             PVE::Storage::vdisk_free($storecfg, $vmstate);
6034         }
6035         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6036         PVE::QemuConfig->write_config($vmid, $conf);
6037     }
6038
6039     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6040
6041     my ($current_machine, $is_deprecated) =
6042         PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6043     if ($is_deprecated) {
6044         log_warn(
6045             "current machine version '$current_machine' is deprecated - see the documentation and ".
6046             "change to a newer one",
6047         );
6048     }
6049
6050     return $res;
6051 }
6052
6053 sub vm_commandline {
6054     my ($storecfg, $vmid, $snapname) = @_;
6055
6056     my $conf = PVE::QemuConfig->load_config($vmid);
6057
6058     my ($forcemachine, $forcecpu);
6059     if ($snapname) {
6060         my $snapshot = $conf->{snapshots}->{$snapname};
6061         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6062
6063         # check for machine or CPU overrides in snapshot
6064         $forcemachine = $snapshot->{runningmachine};
6065         $forcecpu = $snapshot->{runningcpu};
6066
6067         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6068
6069         $conf = $snapshot;
6070     }
6071
6072     my $defaults = load_defaults();
6073
6074     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6075
6076     return PVE::Tools::cmd2string($cmd);
6077 }
6078
6079 sub vm_reset {
6080     my ($vmid, $skiplock) = @_;
6081
6082     PVE::QemuConfig->lock_config($vmid, sub {
6083
6084         my $conf = PVE::QemuConfig->load_config($vmid);
6085
6086         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6087
6088         mon_cmd($vmid, "system_reset");
6089     });
6090 }
6091
6092 sub get_vm_volumes {
6093     my ($conf) = @_;
6094
6095     my $vollist = [];
6096     foreach_volid($conf, sub {
6097         my ($volid, $attr) = @_;
6098
6099         return if $volid =~ m|^/|;
6100
6101         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6102         return if !$sid;
6103
6104         push @$vollist, $volid;
6105     });
6106
6107     return $vollist;
6108 }
6109
6110 sub cleanup_pci_devices {
6111     my ($vmid, $conf) = @_;
6112
6113     foreach my $key (keys %$conf) {
6114         next if $key !~ m/^hostpci(\d+)$/;
6115         my $hostpciindex = $1;
6116         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6117         my $d = parse_hostpci($conf->{$key});
6118         if ($d->{mdev}) {
6119             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6120             # don't want to break ABI just for this two liner
6121             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6122
6123             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6124             # out when we do it first. so wait for 10 seconds and then try it
6125             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/) {
6126                 sleep 10;
6127             }
6128
6129             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6130         }
6131     }
6132     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6133 }
6134
6135 sub vm_stop_cleanup {
6136     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6137
6138     eval {
6139
6140         if (!$keepActive) {
6141             my $vollist = get_vm_volumes($conf);
6142             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6143
6144             if (my $tpmdrive = $conf->{tpmstate0}) {
6145                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6146                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6147                 if ($storeid) {
6148                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6149                 }
6150             }
6151         }
6152
6153         foreach my $ext (qw(mon qmp pid vnc qga)) {
6154             unlink "/var/run/qemu-server/${vmid}.$ext";
6155         }
6156
6157         if ($conf->{ivshmem}) {
6158             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6159             # just delete it for now, VMs which have this already open do not
6160             # are affected, but new VMs will get a separated one. If this
6161             # becomes an issue we either add some sort of ref-counting or just
6162             # add a "don't delete on stop" flag to the ivshmem format.
6163             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6164         }
6165
6166         cleanup_pci_devices($vmid, $conf);
6167
6168         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6169     };
6170     warn $@ if $@; # avoid errors - just warn
6171 }
6172
6173 # call only in locked context
6174 sub _do_vm_stop {
6175     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6176
6177     my $pid = check_running($vmid, $nocheck);
6178     return if !$pid;
6179
6180     my $conf;
6181     if (!$nocheck) {
6182         $conf = PVE::QemuConfig->load_config($vmid);
6183         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6184         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6185             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6186             $timeout = $opts->{down} if $opts->{down};
6187         }
6188         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6189     }
6190
6191     eval {
6192         if ($shutdown) {
6193             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6194                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6195             } else {
6196                 mon_cmd($vmid, "system_powerdown");
6197             }
6198         } else {
6199             mon_cmd($vmid, "quit");
6200         }
6201     };
6202     my $err = $@;
6203
6204     if (!$err) {
6205         $timeout = 60 if !defined($timeout);
6206
6207         my $count = 0;
6208         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6209             $count++;
6210             sleep 1;
6211         }
6212
6213         if ($count >= $timeout) {
6214             if ($force) {
6215                 warn "VM still running - terminating now with SIGTERM\n";
6216                 kill 15, $pid;
6217             } else {
6218                 die "VM quit/powerdown failed - got timeout\n";
6219             }
6220         } else {
6221             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6222             return;
6223         }
6224     } else {
6225        if (!check_running($vmid, $nocheck)) {
6226             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6227             return;
6228        }
6229        if ($force) {
6230             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6231             kill 15, $pid;
6232         } else {
6233             die "VM quit/powerdown failed\n";
6234         }
6235     }
6236
6237     # wait again
6238     $timeout = 10;
6239
6240     my $count = 0;
6241     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6242         $count++;
6243         sleep 1;
6244     }
6245
6246     if ($count >= $timeout) {
6247         warn "VM still running - terminating now with SIGKILL\n";
6248         kill 9, $pid;
6249         sleep 1;
6250     }
6251
6252     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6253 }
6254
6255 # Note: use $nocheck to skip tests if VM configuration file exists.
6256 # We need that when migration VMs to other nodes (files already moved)
6257 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6258 sub vm_stop {
6259     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6260
6261     $force = 1 if !defined($force) && !$shutdown;
6262
6263     if ($migratedfrom){
6264         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6265         kill 15, $pid if $pid;
6266         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6267         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6268         return;
6269     }
6270
6271     PVE::QemuConfig->lock_config($vmid, sub {
6272         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6273    });
6274 }
6275
6276 sub vm_reboot {
6277     my ($vmid, $timeout) = @_;
6278
6279     PVE::QemuConfig->lock_config($vmid, sub {
6280         eval {
6281
6282             # only reboot if running, as qmeventd starts it again on a stop event
6283             return if !check_running($vmid);
6284
6285             create_reboot_request($vmid);
6286
6287             my $storecfg = PVE::Storage::config();
6288             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6289
6290         };
6291         if (my $err = $@) {
6292             # avoid that the next normal shutdown will be confused for a reboot
6293             clear_reboot_request($vmid);
6294             die $err;
6295         }
6296    });
6297 }
6298
6299 # note: if using the statestorage parameter, the caller has to check privileges
6300 sub vm_suspend {
6301     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6302
6303     my $conf;
6304     my $path;
6305     my $storecfg;
6306     my $vmstate;
6307
6308     PVE::QemuConfig->lock_config($vmid, sub {
6309
6310         $conf = PVE::QemuConfig->load_config($vmid);
6311
6312         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6313         PVE::QemuConfig->check_lock($conf)
6314             if !($skiplock || $is_backing_up);
6315
6316         die "cannot suspend to disk during backup\n"
6317             if $is_backing_up && $includestate;
6318
6319         if ($includestate) {
6320             $conf->{lock} = 'suspending';
6321             my $date = strftime("%Y-%m-%d", localtime(time()));
6322             $storecfg = PVE::Storage::config();
6323             if (!$statestorage) {
6324                 $statestorage = find_vmstate_storage($conf, $storecfg);
6325                 # check permissions for the storage
6326                 my $rpcenv = PVE::RPCEnvironment::get();
6327                 if ($rpcenv->{type} ne 'cli') {
6328                     my $authuser = $rpcenv->get_user();
6329                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6330                 }
6331             }
6332
6333
6334             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6335                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6336             $path = PVE::Storage::path($storecfg, $vmstate);
6337             PVE::QemuConfig->write_config($vmid, $conf);
6338         } else {
6339             mon_cmd($vmid, "stop");
6340         }
6341     });
6342
6343     if ($includestate) {
6344         # save vm state
6345         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6346
6347         eval {
6348             set_migration_caps($vmid, 1);
6349             mon_cmd($vmid, "savevm-start", statefile => $path);
6350             for(;;) {
6351                 my $state = mon_cmd($vmid, "query-savevm");
6352                 if (!$state->{status}) {
6353                     die "savevm not active\n";
6354                 } elsif ($state->{status} eq 'active') {
6355                     sleep(1);
6356                     next;
6357                 } elsif ($state->{status} eq 'completed') {
6358                     print "State saved, quitting\n";
6359                     last;
6360                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6361                     die "query-savevm failed with error '$state->{error}'\n"
6362                 } else {
6363                     die "query-savevm returned status '$state->{status}'\n";
6364                 }
6365             }
6366         };
6367         my $err = $@;
6368
6369         PVE::QemuConfig->lock_config($vmid, sub {
6370             $conf = PVE::QemuConfig->load_config($vmid);
6371             if ($err) {
6372                 # cleanup, but leave suspending lock, to indicate something went wrong
6373                 eval {
6374                     mon_cmd($vmid, "savevm-end");
6375                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6376                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6377                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6378                     PVE::QemuConfig->write_config($vmid, $conf);
6379                 };
6380                 warn $@ if $@;
6381                 die $err;
6382             }
6383
6384             die "lock changed unexpectedly\n"
6385                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6386
6387             mon_cmd($vmid, "quit");
6388             $conf->{lock} = 'suspended';
6389             PVE::QemuConfig->write_config($vmid, $conf);
6390         });
6391     }
6392 }
6393
6394 # $nocheck is set when called as part of a migration - in this context the
6395 # location of the config file (source or target node) is not deterministic,
6396 # since migration cannot wait for pmxcfs to process the rename
6397 sub vm_resume {
6398     my ($vmid, $skiplock, $nocheck) = @_;
6399
6400     PVE::QemuConfig->lock_config($vmid, sub {
6401         my $res = mon_cmd($vmid, 'query-status');
6402         my $resume_cmd = 'cont';
6403         my $reset = 0;
6404         my $conf;
6405         if ($nocheck) {
6406             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6407             if ($@) {
6408                 my $vmlist = PVE::Cluster::get_vmlist();
6409                 if (exists($vmlist->{ids}->{$vmid})) {
6410                     my $node = $vmlist->{ids}->{$vmid}->{node};
6411                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6412                 }
6413                 if (!$conf) {
6414                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6415                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6416                 }
6417             }
6418         } else {
6419             $conf = PVE::QemuConfig->load_config($vmid);
6420         }
6421
6422         if ($res->{status}) {
6423             return if $res->{status} eq 'running'; # job done, go home
6424             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6425             $reset = 1 if $res->{status} eq 'shutdown';
6426         }
6427
6428         if (!$nocheck) {
6429             PVE::QemuConfig->check_lock($conf)
6430                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6431         }
6432
6433         if ($reset) {
6434             # required if a VM shuts down during a backup and we get a resume
6435             # request before the backup finishes for example
6436             mon_cmd($vmid, "system_reset");
6437         }
6438
6439         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6440
6441         mon_cmd($vmid, $resume_cmd);
6442     });
6443 }
6444
6445 sub vm_sendkey {
6446     my ($vmid, $skiplock, $key) = @_;
6447
6448     PVE::QemuConfig->lock_config($vmid, sub {
6449
6450         my $conf = PVE::QemuConfig->load_config($vmid);
6451
6452         # there is no qmp command, so we use the human monitor command
6453         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6454         die $res if $res ne '';
6455     });
6456 }
6457
6458 sub check_bridge_access {
6459     my ($rpcenv, $authuser, $conf) = @_;
6460
6461     return 1 if $authuser eq 'root@pam';
6462
6463     for my $opt (sort keys $conf->%*) {
6464         next if $opt !~ m/^net\d+$/;
6465         my $net = parse_net($conf->{$opt});
6466         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6467         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6468     }
6469     return 1;
6470 };
6471
6472 sub check_mapping_access {
6473     my ($rpcenv, $user, $conf) = @_;
6474
6475     for my $opt (keys $conf->%*) {
6476         if ($opt =~ m/^usb\d+$/) {
6477             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6478             if (my $host = $device->{host}) {
6479                 die "only root can set '$opt' config for real devices\n"
6480                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6481             } elsif ($device->{mapping}) {
6482                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6483             } else {
6484                 die "either 'host' or 'mapping' must be set.\n";
6485             }
6486         } elsif ($opt =~ m/^hostpci\d+$/) {
6487             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6488             if ($device->{host}) {
6489                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6490             } elsif ($device->{mapping}) {
6491                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6492             } else {
6493                 die "either 'host' or 'mapping' must be set.\n";
6494             }
6495        }
6496    }
6497 };
6498
6499 sub check_restore_permissions {
6500     my ($rpcenv, $user, $conf) = @_;
6501
6502     check_bridge_access($rpcenv, $user, $conf);
6503     check_mapping_access($rpcenv, $user, $conf);
6504 }
6505 # vzdump restore implementaion
6506
6507 sub tar_archive_read_firstfile {
6508     my $archive = shift;
6509
6510     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6511
6512     # try to detect archive type first
6513     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6514         die "unable to open file '$archive'\n";
6515     my $firstfile = <$fh>;
6516     kill 15, $pid;
6517     close $fh;
6518
6519     die "ERROR: archive contaions no data\n" if !$firstfile;
6520     chomp $firstfile;
6521
6522     return $firstfile;
6523 }
6524
6525 sub tar_restore_cleanup {
6526     my ($storecfg, $statfile) = @_;
6527
6528     print STDERR "starting cleanup\n";
6529
6530     if (my $fd = IO::File->new($statfile, "r")) {
6531         while (defined(my $line = <$fd>)) {
6532             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6533                 my $volid = $2;
6534                 eval {
6535                     if ($volid =~ m|^/|) {
6536                         unlink $volid || die 'unlink failed\n';
6537                     } else {
6538                         PVE::Storage::vdisk_free($storecfg, $volid);
6539                     }
6540                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6541                 };
6542                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6543             } else {
6544                 print STDERR "unable to parse line in statfile - $line";
6545             }
6546         }
6547         $fd->close();
6548     }
6549 }
6550
6551 sub restore_file_archive {
6552     my ($archive, $vmid, $user, $opts) = @_;
6553
6554     return restore_vma_archive($archive, $vmid, $user, $opts)
6555         if $archive eq '-';
6556
6557     my $info = PVE::Storage::archive_info($archive);
6558     my $format = $opts->{format} // $info->{format};
6559     my $comp = $info->{compression};
6560
6561     # try to detect archive format
6562     if ($format eq 'tar') {
6563         return restore_tar_archive($archive, $vmid, $user, $opts);
6564     } else {
6565         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6566     }
6567 }
6568
6569 # hepler to remove disks that will not be used after restore
6570 my $restore_cleanup_oldconf = sub {
6571     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6572
6573     my $kept_disks = {};
6574
6575     PVE::QemuConfig->foreach_volume($oldconf, sub {
6576         my ($ds, $drive) = @_;
6577
6578         return if drive_is_cdrom($drive, 1);
6579
6580         my $volid = $drive->{file};
6581         return if !$volid || $volid =~ m|^/|;
6582
6583         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6584         return if !$path || !$owner || ($owner != $vmid);
6585
6586         # Note: only delete disk we want to restore
6587         # other volumes will become unused
6588         if ($virtdev_hash->{$ds}) {
6589             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6590             if (my $err = $@) {
6591                 warn $err;
6592             }
6593         } else {
6594             $kept_disks->{$volid} = 1;
6595         }
6596     });
6597
6598     # after the restore we have no snapshots anymore
6599     for my $snapname (keys $oldconf->{snapshots}->%*) {
6600         my $snap = $oldconf->{snapshots}->{$snapname};
6601         if ($snap->{vmstate}) {
6602             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6603             if (my $err = $@) {
6604                 warn $err;
6605             }
6606         }
6607
6608         for my $volid (keys $kept_disks->%*) {
6609             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6610             warn $@ if $@;
6611         }
6612     }
6613 };
6614
6615 # Helper to parse vzdump backup device hints
6616 #
6617 # $rpcenv: Environment, used to ckeck storage permissions
6618 # $user: User ID, to check storage permissions
6619 # $storecfg: Storage configuration
6620 # $fh: the file handle for reading the configuration
6621 # $devinfo: should contain device sizes for all backu-up'ed devices
6622 # $options: backup options (pool, default storage)
6623 #
6624 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6625 my $parse_backup_hints = sub {
6626     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6627
6628     my $check_storage = sub { # assert if an image can be allocate
6629         my ($storeid, $scfg) = @_;
6630         die "Content type 'images' is not available on storage '$storeid'\n"
6631             if !$scfg->{content}->{images};
6632         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6633             if $user ne 'root@pam';
6634     };
6635
6636     my $virtdev_hash = {};
6637     while (defined(my $line = <$fh>)) {
6638         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6639             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6640             die "archive does not contain data for drive '$virtdev'\n"
6641                 if !$devinfo->{$devname};
6642
6643             if (defined($options->{storage})) {
6644                 $storeid = $options->{storage} || 'local';
6645             } elsif (!$storeid) {
6646                 $storeid = 'local';
6647             }
6648             $format = 'raw' if !$format;
6649             $devinfo->{$devname}->{devname} = $devname;
6650             $devinfo->{$devname}->{virtdev} = $virtdev;
6651             $devinfo->{$devname}->{format} = $format;
6652             $devinfo->{$devname}->{storeid} = $storeid;
6653
6654             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6655             $check_storage->($storeid, $scfg); # permission and content type check
6656
6657             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6658         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6659             my $virtdev = $1;
6660             my $drive = parse_drive($virtdev, $2);
6661
6662             if (drive_is_cloudinit($drive)) {
6663                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6664                 $storeid = $options->{storage} if defined ($options->{storage});
6665                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6666                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6667
6668                 $check_storage->($storeid, $scfg); # permission and content type check
6669
6670                 $virtdev_hash->{$virtdev} = {
6671                     format => $format,
6672                     storeid => $storeid,
6673                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6674                     is_cloudinit => 1,
6675                 };
6676             }
6677         }
6678     }
6679
6680     return $virtdev_hash;
6681 };
6682
6683 # Helper to allocate and activate all volumes required for a restore
6684 #
6685 # $storecfg: Storage configuration
6686 # $virtdev_hash: as returned by parse_backup_hints()
6687 #
6688 # Returns: { $virtdev => $volid }
6689 my $restore_allocate_devices = sub {
6690     my ($storecfg, $virtdev_hash, $vmid) = @_;
6691
6692     my $map = {};
6693     foreach my $virtdev (sort keys %$virtdev_hash) {
6694         my $d = $virtdev_hash->{$virtdev};
6695         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6696         my $storeid = $d->{storeid};
6697         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6698
6699         # test if requested format is supported
6700         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6701         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6702         $d->{format} = $defFormat if !$supported;
6703
6704         my $name;
6705         if ($d->{is_cloudinit}) {
6706             $name = "vm-$vmid-cloudinit";
6707             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6708             if ($scfg->{path}) {
6709                 $name .= ".$d->{format}";
6710             }
6711         }
6712
6713         my $volid = PVE::Storage::vdisk_alloc(
6714             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6715
6716         print STDERR "new volume ID is '$volid'\n";
6717         $d->{volid} = $volid;
6718
6719         PVE::Storage::activate_volumes($storecfg, [$volid]);
6720
6721         $map->{$virtdev} = $volid;
6722     }
6723
6724     return $map;
6725 };
6726
6727 sub restore_update_config_line {
6728     my ($cookie, $map, $line, $unique) = @_;
6729
6730     return '' if $line =~ m/^\#qmdump\#/;
6731     return '' if $line =~ m/^\#vzdump\#/;
6732     return '' if $line =~ m/^lock:/;
6733     return '' if $line =~ m/^unused\d+:/;
6734     return '' if $line =~ m/^parent:/;
6735
6736     my $res = '';
6737
6738     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6739     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6740         # try to convert old 1.X settings
6741         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6742         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6743             my ($model, $macaddr) = split(/\=/, $devconfig);
6744             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6745             my $net = {
6746                 model => $model,
6747                 bridge => "vmbr$ind",
6748                 macaddr => $macaddr,
6749             };
6750             my $netstr = print_net($net);
6751
6752             $res .= "net$cookie->{netcount}: $netstr\n";
6753             $cookie->{netcount}++;
6754         }
6755     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6756         my ($id, $netstr) = ($1, $2);
6757         my $net = parse_net($netstr);
6758         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6759         $netstr = print_net($net);
6760         $res .= "$id: $netstr\n";
6761     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6762         my $virtdev = $1;
6763         my $value = $3;
6764         my $di = parse_drive($virtdev, $value);
6765         if (defined($di->{backup}) && !$di->{backup}) {
6766             $res .= "#$line";
6767         } elsif ($map->{$virtdev}) {
6768             delete $di->{format}; # format can change on restore
6769             $di->{file} = $map->{$virtdev};
6770             $value = print_drive($di);
6771             $res .= "$virtdev: $value\n";
6772         } else {
6773             $res .= $line;
6774         }
6775     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6776         my $vmgenid = $1;
6777         if ($vmgenid ne '0') {
6778             # always generate a new vmgenid if there was a valid one setup
6779             $vmgenid = generate_uuid();
6780         }
6781         $res .= "vmgenid: $vmgenid\n";
6782     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6783         my ($uuid, $uuid_str);
6784         UUID::generate($uuid);
6785         UUID::unparse($uuid, $uuid_str);
6786         my $smbios1 = parse_smbios1($2);
6787         $smbios1->{uuid} = $uuid_str;
6788         $res .= $1.print_smbios1($smbios1)."\n";
6789     } else {
6790         $res .= $line;
6791     }
6792
6793     return $res;
6794 }
6795
6796 my $restore_deactivate_volumes = sub {
6797     my ($storecfg, $virtdev_hash) = @_;
6798
6799     my $vollist = [];
6800     for my $dev (values $virtdev_hash->%*) {
6801         push $vollist->@*, $dev->{volid} if $dev->{volid};
6802     }
6803
6804     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6805     print STDERR $@ if $@;
6806 };
6807
6808 my $restore_destroy_volumes = sub {
6809     my ($storecfg, $virtdev_hash) = @_;
6810
6811     for my $dev (values $virtdev_hash->%*) {
6812         my $volid = $dev->{volid} or next;
6813         eval {
6814             PVE::Storage::vdisk_free($storecfg, $volid);
6815             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6816         };
6817         print STDERR "unable to cleanup '$volid' - $@" if $@;
6818     }
6819 };
6820
6821 sub restore_merge_config {
6822     my ($filename, $backup_conf_raw, $override_conf) = @_;
6823
6824     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6825     for my $key (keys $override_conf->%*) {
6826         $backup_conf->{$key} = $override_conf->{$key};
6827     }
6828
6829     return $backup_conf;
6830 }
6831
6832 sub scan_volids {
6833     my ($cfg, $vmid) = @_;
6834
6835     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6836
6837     my $volid_hash = {};
6838     foreach my $storeid (keys %$info) {
6839         foreach my $item (@{$info->{$storeid}}) {
6840             next if !($item->{volid} && $item->{size});
6841             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6842             $volid_hash->{$item->{volid}} = $item;
6843         }
6844     }
6845
6846     return $volid_hash;
6847 }
6848
6849 sub update_disk_config {
6850     my ($vmid, $conf, $volid_hash) = @_;
6851
6852     my $changes;
6853     my $prefix = "VM $vmid";
6854
6855     # used and unused disks
6856     my $referenced = {};
6857
6858     # Note: it is allowed to define multiple storages with same path (alias), so
6859     # we need to check both 'volid' and real 'path' (two different volid can point
6860     # to the same path).
6861
6862     my $referencedpath = {};
6863
6864     # update size info
6865     PVE::QemuConfig->foreach_volume($conf, sub {
6866         my ($opt, $drive) = @_;
6867
6868         my $volid = $drive->{file};
6869         return if !$volid;
6870         my $volume = $volid_hash->{$volid};
6871
6872         # mark volid as "in-use" for next step
6873         $referenced->{$volid} = 1;
6874         if ($volume && (my $path = $volume->{path})) {
6875             $referencedpath->{$path} = 1;
6876         }
6877
6878         return if drive_is_cdrom($drive);
6879         return if !$volume;
6880
6881         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6882         if (defined($updated)) {
6883             $changes = 1;
6884             $conf->{$opt} = print_drive($updated);
6885             print "$prefix ($opt): $msg\n";
6886         }
6887     });
6888
6889     # remove 'unusedX' entry if volume is used
6890     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6891         my ($opt, $drive) = @_;
6892
6893         my $volid = $drive->{file};
6894         return if !$volid;
6895
6896         my $path;
6897         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6898         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6899             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6900             $changes = 1;
6901             delete $conf->{$opt};
6902         }
6903
6904         $referenced->{$volid} = 1;
6905         $referencedpath->{$path} = 1 if $path;
6906     });
6907
6908     foreach my $volid (sort keys %$volid_hash) {
6909         next if $volid =~ m/vm-$vmid-state-/;
6910         next if $referenced->{$volid};
6911         my $path = $volid_hash->{$volid}->{path};
6912         next if !$path; # just to be sure
6913         next if $referencedpath->{$path};
6914         $changes = 1;
6915         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6916         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6917         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6918     }
6919
6920     return $changes;
6921 }
6922
6923 sub rescan {
6924     my ($vmid, $nolock, $dryrun) = @_;
6925
6926     my $cfg = PVE::Storage::config();
6927
6928     print "rescan volumes...\n";
6929     my $volid_hash = scan_volids($cfg, $vmid);
6930
6931     my $updatefn =  sub {
6932         my ($vmid) = @_;
6933
6934         my $conf = PVE::QemuConfig->load_config($vmid);
6935
6936         PVE::QemuConfig->check_lock($conf);
6937
6938         my $vm_volids = {};
6939         foreach my $volid (keys %$volid_hash) {
6940             my $info = $volid_hash->{$volid};
6941             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6942         }
6943
6944         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6945
6946         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6947     };
6948
6949     if (defined($vmid)) {
6950         if ($nolock) {
6951             &$updatefn($vmid);
6952         } else {
6953             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6954         }
6955     } else {
6956         my $vmlist = config_list();
6957         foreach my $vmid (keys %$vmlist) {
6958             if ($nolock) {
6959                 &$updatefn($vmid);
6960             } else {
6961                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6962             }
6963         }
6964     }
6965 }
6966
6967 sub restore_proxmox_backup_archive {
6968     my ($archive, $vmid, $user, $options) = @_;
6969
6970     my $storecfg = PVE::Storage::config();
6971
6972     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6973     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6974
6975     my $fingerprint = $scfg->{fingerprint};
6976     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6977
6978     my $repo = PVE::PBSClient::get_repository($scfg);
6979     my $namespace = $scfg->{namespace};
6980
6981     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6982     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6983     local $ENV{PBS_PASSWORD} = $password;
6984     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6985
6986     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6987         PVE::Storage::parse_volname($storecfg, $archive);
6988
6989     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6990
6991     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6992
6993     my $tmpdir = "/var/tmp/vzdumptmp$$";
6994     rmtree $tmpdir;
6995     mkpath $tmpdir;
6996
6997     my $conffile = PVE::QemuConfig->config_file($vmid);
6998      # disable interrupts (always do cleanups)
6999     local $SIG{INT} =
7000         local $SIG{TERM} =
7001         local $SIG{QUIT} =
7002         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7003
7004     # Note: $oldconf is undef if VM does not exists
7005     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7006     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7007     my $new_conf_raw = '';
7008
7009     my $rpcenv = PVE::RPCEnvironment::get();
7010     my $devinfo = {}; # info about drives included in backup
7011     my $virtdev_hash = {}; # info about allocated drives
7012
7013     eval {
7014         # enable interrupts
7015         local $SIG{INT} =
7016             local $SIG{TERM} =
7017             local $SIG{QUIT} =
7018             local $SIG{HUP} =
7019             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7020
7021         my $cfgfn = "$tmpdir/qemu-server.conf";
7022         my $firewall_config_fn = "$tmpdir/fw.conf";
7023         my $index_fn = "$tmpdir/index.json";
7024
7025         my $cmd = "restore";
7026
7027         my $param = [$pbs_backup_name, "index.json", $index_fn];
7028         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7029         my $index = PVE::Tools::file_get_contents($index_fn);
7030         $index = decode_json($index);
7031
7032         foreach my $info (@{$index->{files}}) {
7033             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7034                 my $devname = $1;
7035                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7036                     $devinfo->{$devname}->{size} = $1;
7037                 } else {
7038                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7039                 }
7040             }
7041         }
7042
7043         my $is_qemu_server_backup = scalar(
7044             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7045         );
7046         if (!$is_qemu_server_backup) {
7047             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7048         }
7049         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7050
7051         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7052         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7053
7054         if ($has_firewall_config) {
7055             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7056             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7057
7058             my $pve_firewall_dir = '/etc/pve/firewall';
7059             mkdir $pve_firewall_dir; # make sure the dir exists
7060             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7061         }
7062
7063         my $fh = IO::File->new($cfgfn, "r") ||
7064             die "unable to read qemu-server.conf - $!\n";
7065
7066         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7067
7068         # fixme: rate limit?
7069
7070         # create empty/temp config
7071         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7072
7073         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7074
7075         # allocate volumes
7076         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7077
7078         foreach my $virtdev (sort keys %$virtdev_hash) {
7079             my $d = $virtdev_hash->{$virtdev};
7080             next if $d->{is_cloudinit}; # no need to restore cloudinit
7081
7082             # this fails if storage is unavailable
7083             my $volid = $d->{volid};
7084             my $path = PVE::Storage::path($storecfg, $volid);
7085
7086             # for live-restore we only want to preload the efidisk and TPM state
7087             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7088
7089             my @ns_arg;
7090             if (defined(my $ns = $scfg->{namespace})) {
7091                 @ns_arg = ('--ns', $ns);
7092             }
7093
7094             my $pbs_restore_cmd = [
7095                 '/usr/bin/pbs-restore',
7096                 '--repository', $repo,
7097                 @ns_arg,
7098                 $pbs_backup_name,
7099                 "$d->{devname}.img.fidx",
7100                 $path,
7101                 '--verbose',
7102                 ];
7103
7104             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7105             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7106
7107             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7108                 push @$pbs_restore_cmd, '--skip-zero';
7109             }
7110
7111             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7112             print "restore proxmox backup image: $dbg_cmdstring\n";
7113             run_command($pbs_restore_cmd);
7114         }
7115
7116         $fh->seek(0, 0) || die "seek failed - $!\n";
7117
7118         my $cookie = { netcount => 0 };
7119         while (defined(my $line = <$fh>)) {
7120             $new_conf_raw .= restore_update_config_line(
7121                 $cookie,
7122                 $map,
7123                 $line,
7124                 $options->{unique},
7125             );
7126         }
7127
7128         $fh->close();
7129     };
7130     my $err = $@;
7131
7132     if ($err || !$options->{live}) {
7133         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7134     }
7135
7136     rmtree $tmpdir;
7137
7138     if ($err) {
7139         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7140         die $err;
7141     }
7142
7143     if ($options->{live}) {
7144         # keep lock during live-restore
7145         $new_conf_raw .= "\nlock: create";
7146     }
7147
7148     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7149     check_restore_permissions($rpcenv, $user, $new_conf);
7150     PVE::QemuConfig->write_config($vmid, $new_conf);
7151
7152     eval { rescan($vmid, 1); };
7153     warn $@ if $@;
7154
7155     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7156
7157     if ($options->{live}) {
7158         # enable interrupts
7159         local $SIG{INT} =
7160             local $SIG{TERM} =
7161             local $SIG{QUIT} =
7162             local $SIG{HUP} =
7163             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7164
7165         my $conf = PVE::QemuConfig->load_config($vmid);
7166         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7167
7168         # these special drives are already restored before start
7169         delete $devinfo->{'drive-efidisk0'};
7170         delete $devinfo->{'drive-tpmstate0-backup'};
7171
7172         my $pbs_opts = {
7173             repo => $repo,
7174             keyfile => $keyfile,
7175             snapshot => $pbs_backup_name,
7176             namespace => $namespace,
7177         };
7178         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7179
7180         PVE::QemuConfig->remove_lock($vmid, "create");
7181     }
7182 }
7183
7184 sub pbs_live_restore {
7185     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7186
7187     print "starting VM for live-restore\n";
7188     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7189
7190     my $pbs_backing = {};
7191     for my $ds (keys %$restored_disks) {
7192         $ds =~ m/^drive-(.*)$/;
7193         my $confname = $1;
7194         $pbs_backing->{$confname} = {
7195             repository => $opts->{repo},
7196             snapshot => $opts->{snapshot},
7197             archive => "$ds.img.fidx",
7198         };
7199         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7200         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7201
7202         my $drive = parse_drive($confname, $conf->{$confname});
7203         print "restoring '$ds' to '$drive->{file}'\n";
7204     }
7205
7206     my $drives_streamed = 0;
7207     eval {
7208         # make sure HA doesn't interrupt our restore by stopping the VM
7209         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7210             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7211         }
7212
7213         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7214         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7215         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7216
7217         my $qmeventd_fd = register_qmeventd_handle($vmid);
7218
7219         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7220         # this will effectively collapse the backing image chain consisting of
7221         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7222         # removes itself once all backing images vanish with 'auto-remove=on')
7223         my $jobs = {};
7224         for my $ds (sort keys %$restored_disks) {
7225             my $job_id = "restore-$ds";
7226             mon_cmd($vmid, 'block-stream',
7227                 'job-id' => $job_id,
7228                 device => "$ds",
7229             );
7230             $jobs->{$job_id} = {};
7231         }
7232
7233         mon_cmd($vmid, 'cont');
7234         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7235
7236         print "restore-drive jobs finished successfully, removing all tracking block devices"
7237             ." to disconnect from Proxmox Backup Server\n";
7238
7239         for my $ds (sort keys %$restored_disks) {
7240             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7241         }
7242
7243         close($qmeventd_fd);
7244     };
7245
7246     my $err = $@;
7247
7248     if ($err) {
7249         warn "An error occurred during live-restore: $err\n";
7250         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7251         die "live-restore failed\n";
7252     }
7253 }
7254
7255 sub restore_vma_archive {
7256     my ($archive, $vmid, $user, $opts, $comp) = @_;
7257
7258     my $readfrom = $archive;
7259
7260     my $cfg = PVE::Storage::config();
7261     my $commands = [];
7262     my $bwlimit = $opts->{bwlimit};
7263
7264     my $dbg_cmdstring = '';
7265     my $add_pipe = sub {
7266         my ($cmd) = @_;
7267         push @$commands, $cmd;
7268         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7269         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7270         $readfrom = '-';
7271     };
7272
7273     my $input = undef;
7274     if ($archive eq '-') {
7275         $input = '<&STDIN';
7276     } else {
7277         # If we use a backup from a PVE defined storage we also consider that
7278         # storage's rate limit:
7279         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7280         if (defined($volid)) {
7281             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7282             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7283             if ($readlimit) {
7284                 print STDERR "applying read rate limit: $readlimit\n";
7285                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7286                 $add_pipe->($cstream);
7287             }
7288         }
7289     }
7290
7291     if ($comp) {
7292         my $info = PVE::Storage::decompressor_info('vma', $comp);
7293         my $cmd = $info->{decompressor};
7294         push @$cmd, $readfrom;
7295         $add_pipe->($cmd);
7296     }
7297
7298     my $tmpdir = "/var/tmp/vzdumptmp$$";
7299     rmtree $tmpdir;
7300
7301     # disable interrupts (always do cleanups)
7302     local $SIG{INT} =
7303         local $SIG{TERM} =
7304         local $SIG{QUIT} =
7305         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7306
7307     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7308     POSIX::mkfifo($mapfifo, 0600);
7309     my $fifofh;
7310     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7311
7312     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7313
7314     my $devinfo = {}; # info about drives included in backup
7315     my $virtdev_hash = {}; # info about allocated drives
7316
7317     my $rpcenv = PVE::RPCEnvironment::get();
7318
7319     my $conffile = PVE::QemuConfig->config_file($vmid);
7320
7321     # Note: $oldconf is undef if VM does not exist
7322     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7323     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7324     my $new_conf_raw = '';
7325
7326     my %storage_limits;
7327
7328     my $print_devmap = sub {
7329         my $cfgfn = "$tmpdir/qemu-server.conf";
7330
7331         # we can read the config - that is already extracted
7332         my $fh = IO::File->new($cfgfn, "r") ||
7333             die "unable to read qemu-server.conf - $!\n";
7334
7335         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7336         if (-f $fwcfgfn) {
7337             my $pve_firewall_dir = '/etc/pve/firewall';
7338             mkdir $pve_firewall_dir; # make sure the dir exists
7339             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7340         }
7341
7342         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7343
7344         foreach my $info (values %{$virtdev_hash}) {
7345             my $storeid = $info->{storeid};
7346             next if defined($storage_limits{$storeid});
7347
7348             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7349             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7350             $storage_limits{$storeid} = $limit * 1024;
7351         }
7352
7353         foreach my $devname (keys %$devinfo) {
7354             die "found no device mapping information for device '$devname'\n"
7355                 if !$devinfo->{$devname}->{virtdev};
7356         }
7357
7358         # create empty/temp config
7359         if ($oldconf) {
7360             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7361             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7362         }
7363
7364         # allocate volumes
7365         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7366
7367         # print restore information to $fifofh
7368         foreach my $virtdev (sort keys %$virtdev_hash) {
7369             my $d = $virtdev_hash->{$virtdev};
7370             next if $d->{is_cloudinit}; # no need to restore cloudinit
7371
7372             my $storeid = $d->{storeid};
7373             my $volid = $d->{volid};
7374
7375             my $map_opts = '';
7376             if (my $limit = $storage_limits{$storeid}) {
7377                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7378             }
7379
7380             my $write_zeros = 1;
7381             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7382                 $write_zeros = 0;
7383             }
7384
7385             my $path = PVE::Storage::path($cfg, $volid);
7386
7387             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7388
7389             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7390         }
7391
7392         $fh->seek(0, 0) || die "seek failed - $!\n";
7393
7394         my $cookie = { netcount => 0 };
7395         while (defined(my $line = <$fh>)) {
7396             $new_conf_raw .= restore_update_config_line(
7397                 $cookie,
7398                 $map,
7399                 $line,
7400                 $opts->{unique},
7401             );
7402         }
7403
7404         $fh->close();
7405     };
7406
7407     my $oldtimeout;
7408
7409     eval {
7410         # enable interrupts
7411         local $SIG{INT} =
7412             local $SIG{TERM} =
7413             local $SIG{QUIT} =
7414             local $SIG{HUP} =
7415             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7416         local $SIG{ALRM} = sub { die "got timeout\n"; };
7417
7418         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7419
7420         my $parser = sub {
7421             my $line = shift;
7422
7423             print "$line\n";
7424
7425             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7426                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7427                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7428             } elsif ($line =~ m/^CTIME: /) {
7429                 # we correctly received the vma config, so we can disable
7430                 # the timeout now for disk allocation
7431                 alarm($oldtimeout || 0);
7432                 $oldtimeout = undef;
7433                 &$print_devmap();
7434                 print $fifofh "done\n";
7435                 close($fifofh);
7436                 $fifofh = undef;
7437             }
7438         };
7439
7440         print "restore vma archive: $dbg_cmdstring\n";
7441         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7442     };
7443     my $err = $@;
7444
7445     alarm($oldtimeout) if $oldtimeout;
7446
7447     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7448
7449     close($fifofh) if $fifofh;
7450     unlink $mapfifo;
7451     rmtree $tmpdir;
7452
7453     if ($err) {
7454         $restore_destroy_volumes->($cfg, $virtdev_hash);
7455         die $err;
7456     }
7457
7458     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7459     check_restore_permissions($rpcenv, $user, $new_conf);
7460     PVE::QemuConfig->write_config($vmid, $new_conf);
7461
7462     eval { rescan($vmid, 1); };
7463     warn $@ if $@;
7464
7465     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7466 }
7467
7468 sub restore_tar_archive {
7469     my ($archive, $vmid, $user, $opts) = @_;
7470
7471     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7472         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7473         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7474     }
7475
7476     if ($archive ne '-') {
7477         my $firstfile = tar_archive_read_firstfile($archive);
7478         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7479             if $firstfile ne 'qemu-server.conf';
7480     }
7481
7482     my $storecfg = PVE::Storage::config();
7483
7484     # avoid zombie disks when restoring over an existing VM -> cleanup first
7485     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7486     # skiplock=1 because qmrestore has set the 'create' lock itself already
7487     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7488     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7489
7490     my $tocmd = "/usr/lib/qemu-server/qmextract";
7491
7492     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7493     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7494     $tocmd .= ' --prealloc' if $opts->{prealloc};
7495     $tocmd .= ' --info' if $opts->{info};
7496
7497     # tar option "xf" does not autodetect compression when read from STDIN,
7498     # so we pipe to zcat
7499     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7500         PVE::Tools::shellquote("--to-command=$tocmd");
7501
7502     my $tmpdir = "/var/tmp/vzdumptmp$$";
7503     mkpath $tmpdir;
7504
7505     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7506     local $ENV{VZDUMP_VMID} = $vmid;
7507     local $ENV{VZDUMP_USER} = $user;
7508
7509     my $conffile = PVE::QemuConfig->config_file($vmid);
7510     my $new_conf_raw = '';
7511
7512     # disable interrupts (always do cleanups)
7513     local $SIG{INT} =
7514         local $SIG{TERM} =
7515         local $SIG{QUIT} =
7516         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7517
7518     eval {
7519         # enable interrupts
7520         local $SIG{INT} =
7521             local $SIG{TERM} =
7522             local $SIG{QUIT} =
7523             local $SIG{HUP} =
7524             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7525
7526         if ($archive eq '-') {
7527             print "extracting archive from STDIN\n";
7528             run_command($cmd, input => "<&STDIN");
7529         } else {
7530             print "extracting archive '$archive'\n";
7531             run_command($cmd);
7532         }
7533
7534         return if $opts->{info};
7535
7536         # read new mapping
7537         my $map = {};
7538         my $statfile = "$tmpdir/qmrestore.stat";
7539         if (my $fd = IO::File->new($statfile, "r")) {
7540             while (defined (my $line = <$fd>)) {
7541                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7542                     $map->{$1} = $2 if $1;
7543                 } else {
7544                     print STDERR "unable to parse line in statfile - $line\n";
7545                 }
7546             }
7547             $fd->close();
7548         }
7549
7550         my $confsrc = "$tmpdir/qemu-server.conf";
7551
7552         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7553
7554         my $cookie = { netcount => 0 };
7555         while (defined (my $line = <$srcfd>)) {
7556             $new_conf_raw .= restore_update_config_line(
7557                 $cookie,
7558                 $map,
7559                 $line,
7560                 $opts->{unique},
7561             );
7562         }
7563
7564         $srcfd->close();
7565     };
7566     if (my $err = $@) {
7567         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7568         die $err;
7569     }
7570
7571     rmtree $tmpdir;
7572
7573     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7574
7575     PVE::Cluster::cfs_update(); # make sure we read new file
7576
7577     eval { rescan($vmid, 1); };
7578     warn $@ if $@;
7579 };
7580
7581 sub foreach_storage_used_by_vm {
7582     my ($conf, $func) = @_;
7583
7584     my $sidhash = {};
7585
7586     PVE::QemuConfig->foreach_volume($conf, sub {
7587         my ($ds, $drive) = @_;
7588         return if drive_is_cdrom($drive);
7589
7590         my $volid = $drive->{file};
7591
7592         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7593         $sidhash->{$sid} = $sid if $sid;
7594     });
7595
7596     foreach my $sid (sort keys %$sidhash) {
7597         &$func($sid);
7598     }
7599 }
7600
7601 my $qemu_snap_storage = {
7602     rbd => 1,
7603 };
7604 sub do_snapshots_with_qemu {
7605     my ($storecfg, $volid, $deviceid) = @_;
7606
7607     return if $deviceid =~ m/tpmstate0/;
7608
7609     my $storage_name = PVE::Storage::parse_volume_id($volid);
7610     my $scfg = $storecfg->{ids}->{$storage_name};
7611     die "could not find storage '$storage_name'\n" if !defined($scfg);
7612
7613     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7614         return 1;
7615     }
7616
7617     if ($volid =~ m/\.(qcow2|qed)$/){
7618         return 1;
7619     }
7620
7621     return;
7622 }
7623
7624 sub qga_check_running {
7625     my ($vmid, $nowarn) = @_;
7626
7627     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7628     if ($@) {
7629         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7630         return 0;
7631     }
7632     return 1;
7633 }
7634
7635 sub template_create {
7636     my ($vmid, $conf, $disk) = @_;
7637
7638     my $storecfg = PVE::Storage::config();
7639
7640     PVE::QemuConfig->foreach_volume($conf, sub {
7641         my ($ds, $drive) = @_;
7642
7643         return if drive_is_cdrom($drive);
7644         return if $disk && $ds ne $disk;
7645
7646         my $volid = $drive->{file};
7647         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7648
7649         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7650         $drive->{file} = $voliddst;
7651         $conf->{$ds} = print_drive($drive);
7652         PVE::QemuConfig->write_config($vmid, $conf);
7653     });
7654 }
7655
7656 sub convert_iscsi_path {
7657     my ($path) = @_;
7658
7659     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7660         my $portal = $1;
7661         my $target = $2;
7662         my $lun = $3;
7663
7664         my $initiator_name = get_initiator_name();
7665
7666         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7667                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7668     }
7669
7670     die "cannot convert iscsi path '$path', unkown format\n";
7671 }
7672
7673 sub qemu_img_convert {
7674     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7675
7676     my $storecfg = PVE::Storage::config();
7677     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7678     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7679
7680     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7681
7682     my $cachemode;
7683     my $src_path;
7684     my $src_is_iscsi = 0;
7685     my $src_format;
7686
7687     if ($src_storeid) {
7688         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7689         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7690         $src_format = qemu_img_format($src_scfg, $src_volname);
7691         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7692         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7693         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7694     } elsif (-f $src_volid || -b $src_volid) {
7695         $src_path = $src_volid;
7696         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7697             $src_format = $1;
7698         }
7699     }
7700
7701     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7702
7703     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7704     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7705     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7706     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7707
7708     my $cmd = [];
7709     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7710     push @$cmd, '-l', "snapshot.name=$snapname"
7711         if $snapname && $src_format && $src_format eq "qcow2";
7712     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7713     push @$cmd, '-T', $cachemode if defined($cachemode);
7714     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7715
7716     if ($src_is_iscsi) {
7717         push @$cmd, '--image-opts';
7718         $src_path = convert_iscsi_path($src_path);
7719     } elsif ($src_format) {
7720         push @$cmd, '-f', $src_format;
7721     }
7722
7723     if ($dst_is_iscsi) {
7724         push @$cmd, '--target-image-opts';
7725         $dst_path = convert_iscsi_path($dst_path);
7726     } else {
7727         push @$cmd, '-O', $dst_format;
7728     }
7729
7730     push @$cmd, $src_path;
7731
7732     if (!$dst_is_iscsi && $is_zero_initialized) {
7733         push @$cmd, "zeroinit:$dst_path";
7734     } else {
7735         push @$cmd, $dst_path;
7736     }
7737
7738     my $parser = sub {
7739         my $line = shift;
7740         if($line =~ m/\((\S+)\/100\%\)/){
7741             my $percent = $1;
7742             my $transferred = int($size * $percent / 100);
7743             my $total_h = render_bytes($size, 1);
7744             my $transferred_h = render_bytes($transferred, 1);
7745
7746             print "transferred $transferred_h of $total_h ($percent%)\n";
7747         }
7748
7749     };
7750
7751     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7752     my $err = $@;
7753     die "copy failed: $err" if $err;
7754 }
7755
7756 sub qemu_img_format {
7757     my ($scfg, $volname) = @_;
7758
7759     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7760         return $1;
7761     } else {
7762         return "raw";
7763     }
7764 }
7765
7766 sub qemu_drive_mirror {
7767     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7768
7769     $jobs = {} if !$jobs;
7770
7771     my $qemu_target;
7772     my $format;
7773     $jobs->{"drive-$drive"} = {};
7774
7775     if ($dst_volid =~ /^nbd:/) {
7776         $qemu_target = $dst_volid;
7777         $format = "nbd";
7778     } else {
7779         my $storecfg = PVE::Storage::config();
7780         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7781
7782         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7783
7784         $format = qemu_img_format($dst_scfg, $dst_volname);
7785
7786         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7787
7788         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7789     }
7790
7791     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7792     $opts->{format} = $format if $format;
7793
7794     if (defined($src_bitmap)) {
7795         $opts->{sync} = 'incremental';
7796         $opts->{bitmap} = $src_bitmap;
7797         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7798     }
7799
7800     if (defined($bwlimit)) {
7801         $opts->{speed} = $bwlimit * 1024;
7802         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7803     } else {
7804         print "drive mirror is starting for drive-$drive\n";
7805     }
7806
7807     # if a job already runs for this device we get an error, catch it for cleanup
7808     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7809     if (my $err = $@) {
7810         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7811         warn "$@\n" if $@;
7812         die "mirroring error: $err\n";
7813     }
7814
7815     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7816 }
7817
7818 # $completion can be either
7819 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7820 # 'cancel': wait until all jobs are ready, block-job-cancel them
7821 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7822 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7823 sub qemu_drive_mirror_monitor {
7824     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7825
7826     $completion //= 'complete';
7827     $op //= "mirror";
7828
7829     eval {
7830         my $err_complete = 0;
7831
7832         my $starttime = time ();
7833         while (1) {
7834             die "block job ('$op') timed out\n" if $err_complete > 300;
7835
7836             my $stats = mon_cmd($vmid, "query-block-jobs");
7837             my $ctime = time();
7838
7839             my $running_jobs = {};
7840             for my $stat (@$stats) {
7841                 next if $stat->{type} ne $op;
7842                 $running_jobs->{$stat->{device}} = $stat;
7843             }
7844
7845             my $readycounter = 0;
7846
7847             for my $job_id (sort keys %$jobs) {
7848                 my $job = $running_jobs->{$job_id};
7849
7850                 my $vanished = !defined($job);
7851                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7852                 if($complete || ($vanished && $completion eq 'auto')) {
7853                     print "$job_id: $op-job finished\n";
7854                     delete $jobs->{$job_id};
7855                     next;
7856                 }
7857
7858                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7859
7860                 my $busy = $job->{busy};
7861                 my $ready = $job->{ready};
7862                 if (my $total = $job->{len}) {
7863                     my $transferred = $job->{offset} || 0;
7864                     my $remaining = $total - $transferred;
7865                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7866
7867                     my $duration = $ctime - $starttime;
7868                     my $total_h = render_bytes($total, 1);
7869                     my $transferred_h = render_bytes($transferred, 1);
7870
7871                     my $status = sprintf(
7872                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7873                         render_duration($duration),
7874                     );
7875
7876                     if ($ready) {
7877                         if ($busy) {
7878                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7879                         } else {
7880                             $status .= ", ready";
7881                         }
7882                     }
7883                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7884                     $jobs->{$job_id}->{ready} = $ready;
7885                 }
7886
7887                 $readycounter++ if $job->{ready};
7888             }
7889
7890             last if scalar(keys %$jobs) == 0;
7891
7892             if ($readycounter == scalar(keys %$jobs)) {
7893                 print "all '$op' jobs are ready\n";
7894
7895                 # do the complete later (or has already been done)
7896                 last if $completion eq 'skip' || $completion eq 'auto';
7897
7898                 if ($vmiddst && $vmiddst != $vmid) {
7899                     my $agent_running = $qga && qga_check_running($vmid);
7900                     if ($agent_running) {
7901                         print "freeze filesystem\n";
7902                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7903                         warn $@ if $@;
7904                     } else {
7905                         print "suspend vm\n";
7906                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7907                         warn $@ if $@;
7908                     }
7909
7910                     # if we clone a disk for a new target vm, we don't switch the disk
7911                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7912
7913                     if ($agent_running) {
7914                         print "unfreeze filesystem\n";
7915                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7916                         warn $@ if $@;
7917                     } else {
7918                         print "resume vm\n";
7919                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7920                         warn $@ if $@;
7921                     }
7922
7923                     last;
7924                 } else {
7925
7926                     for my $job_id (sort keys %$jobs) {
7927                         # try to switch the disk if source and destination are on the same guest
7928                         print "$job_id: Completing block job_id...\n";
7929
7930                         my $op;
7931                         if ($completion eq 'complete') {
7932                             $op = 'block-job-complete';
7933                         } elsif ($completion eq 'cancel') {
7934                             $op = 'block-job-cancel';
7935                         } else {
7936                             die "invalid completion value: $completion\n";
7937                         }
7938                         eval { mon_cmd($vmid, $op, device => $job_id) };
7939                         if ($@ =~ m/cannot be completed/) {
7940                             print "$job_id: block job cannot be completed, trying again.\n";
7941                             $err_complete++;
7942                         }else {
7943                             print "$job_id: Completed successfully.\n";
7944                             $jobs->{$job_id}->{complete} = 1;
7945                         }
7946                     }
7947                 }
7948             }
7949             sleep 1;
7950         }
7951     };
7952     my $err = $@;
7953
7954     if ($err) {
7955         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7956         die "block job ($op) error: $err";
7957     }
7958 }
7959
7960 sub qemu_blockjobs_cancel {
7961     my ($vmid, $jobs) = @_;
7962
7963     foreach my $job (keys %$jobs) {
7964         print "$job: Cancelling block job\n";
7965         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7966         $jobs->{$job}->{cancel} = 1;
7967     }
7968
7969     while (1) {
7970         my $stats = mon_cmd($vmid, "query-block-jobs");
7971
7972         my $running_jobs = {};
7973         foreach my $stat (@$stats) {
7974             $running_jobs->{$stat->{device}} = $stat;
7975         }
7976
7977         foreach my $job (keys %$jobs) {
7978
7979             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7980                 print "$job: Done.\n";
7981                 delete $jobs->{$job};
7982             }
7983         }
7984
7985         last if scalar(keys %$jobs) == 0;
7986
7987         sleep 1;
7988     }
7989 }
7990
7991 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
7992 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
7993 my sub clone_disk_check_io_uring {
7994     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
7995
7996     return if !$use_drive_mirror;
7997
7998     # Don't complain when not changing storage.
7999     # Assume if it works for the source, it'll work for the target too.
8000     return if $src_storeid eq $dst_storeid;
8001
8002     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8003     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8004
8005     my $cache_direct = drive_uses_cache_direct($src_drive);
8006
8007     my $src_uses_io_uring;
8008     if ($src_drive->{aio}) {
8009         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8010     } else {
8011         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8012     }
8013
8014     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8015         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8016 }
8017
8018 sub clone_disk {
8019     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8020
8021     my ($vmid, $running) = $source->@{qw(vmid running)};
8022     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8023
8024     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8025     my ($storage, $format) = $dest->@{qw(storage format)};
8026
8027     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8028
8029     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8030         die "cloning from/to EFI disk requires EFI disk\n"
8031             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8032         die "cloning from/to TPM state requires TPM state\n"
8033             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8034
8035         # This would lead to two device nodes in QEMU pointing to the same backing image!
8036         die "cannot change drive name when cloning disk from/to the same VM\n"
8037             if $use_drive_mirror && $vmid == $newvmid;
8038     }
8039
8040     die "cannot move TPM state while VM is running\n"
8041         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8042
8043     my $newvolid;
8044
8045     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8046     print "$src_drivename " if $src_drivename;
8047     print "($drive->{file})\n";
8048
8049     if (!$full) {
8050         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8051         push @$newvollist, $newvolid;
8052     } else {
8053         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8054         my $storeid = $storage || $src_storeid;
8055
8056         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8057
8058         my $name = undef;
8059         my $size = undef;
8060         if (drive_is_cloudinit($drive)) {
8061             $name = "vm-$newvmid-cloudinit";
8062             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8063             if ($scfg->{path}) {
8064                 $name .= ".$dst_format";
8065             }
8066             $snapname = undef;
8067             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8068         } elsif ($dst_drivename eq 'efidisk0') {
8069             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8070         } elsif ($dst_drivename eq 'tpmstate0') {
8071             $dst_format = 'raw';
8072             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8073         } else {
8074             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8075
8076             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8077         }
8078         $newvolid = PVE::Storage::vdisk_alloc(
8079             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8080         );
8081         push @$newvollist, $newvolid;
8082
8083         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8084
8085         if (drive_is_cloudinit($drive)) {
8086             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8087             # if this is the case, we have to complete any block-jobs still there from
8088             # previous drive-mirrors
8089             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8090                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8091             }
8092             goto no_data_clone;
8093         }
8094
8095         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8096         if ($use_drive_mirror) {
8097             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8098                 $completion, $qga, $bwlimit);
8099         } else {
8100             if ($dst_drivename eq 'efidisk0') {
8101                 # the relevant data on the efidisk may be smaller than the source
8102                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8103                 # that is given by the OVMF_VARS.fd
8104                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8105                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8106
8107                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8108
8109                 # better for Ceph if block size is not too small, see bug #3324
8110                 my $bs = 1024*1024;
8111
8112                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8113
8114                 if ($src_format eq 'qcow2' && $snapname) {
8115                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8116                         if !min_version(kvm_user_version(), 6, 2);
8117                     push $cmd->@*, '-l', $snapname;
8118                 }
8119                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8120                 run_command($cmd);
8121             } else {
8122                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8123             }
8124         }
8125     }
8126
8127 no_data_clone:
8128     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8129
8130     my $disk = dclone($drive);
8131     delete $disk->{format};
8132     $disk->{file} = $newvolid;
8133     $disk->{size} = $size if defined($size);
8134
8135     return $disk;
8136 }
8137
8138 sub get_running_qemu_version {
8139     my ($vmid) = @_;
8140     my $res = mon_cmd($vmid, "query-version");
8141     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8142 }
8143
8144 sub qemu_use_old_bios_files {
8145     my ($machine_type) = @_;
8146
8147     return if !$machine_type;
8148
8149     my $use_old_bios_files = undef;
8150
8151     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8152         $machine_type = $1;
8153         $use_old_bios_files = 1;
8154     } else {
8155         my $version = extract_version($machine_type, kvm_user_version());
8156         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8157         # load new efi bios files on migration. So this hack is required to allow
8158         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8159         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8160         $use_old_bios_files = !min_version($version, 2, 4);
8161     }
8162
8163     return ($use_old_bios_files, $machine_type);
8164 }
8165
8166 sub get_efivars_size {
8167     my ($conf, $efidisk) = @_;
8168
8169     my $arch = get_vm_arch($conf);
8170     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8171     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8172     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8173     return -s $ovmf_vars;
8174 }
8175
8176 sub update_efidisk_size {
8177     my ($conf) = @_;
8178
8179     return if !defined($conf->{efidisk0});
8180
8181     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8182     $disk->{size} = get_efivars_size($conf);
8183     $conf->{efidisk0} = print_drive($disk);
8184
8185     return;
8186 }
8187
8188 sub update_tpmstate_size {
8189     my ($conf) = @_;
8190
8191     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8192     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8193     $conf->{tpmstate0} = print_drive($disk);
8194 }
8195
8196 sub create_efidisk($$$$$$$) {
8197     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8198
8199     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8200
8201     my $vars_size_b = -s $ovmf_vars;
8202     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8203     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8204     PVE::Storage::activate_volumes($storecfg, [$volid]);
8205
8206     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8207     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8208
8209     return ($volid, $size/1024);
8210 }
8211
8212 sub vm_iothreads_list {
8213     my ($vmid) = @_;
8214
8215     my $res = mon_cmd($vmid, 'query-iothreads');
8216
8217     my $iothreads = {};
8218     foreach my $iothread (@$res) {
8219         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8220     }
8221
8222     return $iothreads;
8223 }
8224
8225 sub scsihw_infos {
8226     my ($conf, $drive) = @_;
8227
8228     my $maxdev = 0;
8229
8230     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8231         $maxdev = 7;
8232     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8233         $maxdev = 1;
8234     } else {
8235         $maxdev = 256;
8236     }
8237
8238     my $controller = int($drive->{index} / $maxdev);
8239     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8240         ? "virtioscsi"
8241         : "scsihw";
8242
8243     return ($maxdev, $controller, $controller_prefix);
8244 }
8245
8246 sub resolve_dst_disk_format {
8247         my ($storecfg, $storeid, $src_volname, $format) = @_;
8248         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8249
8250         if (!$format) {
8251             # if no target format is specified, use the source disk format as hint
8252             if ($src_volname) {
8253                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8254                 $format = qemu_img_format($scfg, $src_volname);
8255             } else {
8256                 return $defFormat;
8257             }
8258         }
8259
8260         # test if requested format is supported - else use default
8261         my $supported = grep { $_ eq $format } @$validFormats;
8262         $format = $defFormat if !$supported;
8263         return $format;
8264 }
8265
8266 # NOTE: if this logic changes, please update docs & possibly gui logic
8267 sub find_vmstate_storage {
8268     my ($conf, $storecfg) = @_;
8269
8270     # first, return storage from conf if set
8271     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8272
8273     my ($target, $shared, $local);
8274
8275     foreach_storage_used_by_vm($conf, sub {
8276         my ($sid) = @_;
8277         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8278         my $dst = $scfg->{shared} ? \$shared : \$local;
8279         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8280     });
8281
8282     # second, use shared storage where VM has at least one disk
8283     # third, use local storage where VM has at least one disk
8284     # fall back to local storage
8285     $target = $shared // $local // 'local';
8286
8287     return $target;
8288 }
8289
8290 sub generate_uuid {
8291     my ($uuid, $uuid_str);
8292     UUID::generate($uuid);
8293     UUID::unparse($uuid, $uuid_str);
8294     return $uuid_str;
8295 }
8296
8297 sub generate_smbios1_uuid {
8298     return "uuid=".generate_uuid();
8299 }
8300
8301 sub nbd_stop {
8302     my ($vmid) = @_;
8303
8304     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8305 }
8306
8307 sub create_reboot_request {
8308     my ($vmid) = @_;
8309     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8310         or die "failed to create reboot trigger file: $!\n";
8311     close($fh);
8312 }
8313
8314 sub clear_reboot_request {
8315     my ($vmid) = @_;
8316     my $path = "/run/qemu-server/$vmid.reboot";
8317     my $res = 0;
8318
8319     $res = unlink($path);
8320     die "could not remove reboot request for $vmid: $!"
8321         if !$res && $! != POSIX::ENOENT;
8322
8323     return $res;
8324 }
8325
8326 sub bootorder_from_legacy {
8327     my ($conf, $bootcfg) = @_;
8328
8329     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8330     my $bootindex_hash = {};
8331     my $i = 1;
8332     foreach my $o (split(//, $boot)) {
8333         $bootindex_hash->{$o} = $i*100;
8334         $i++;
8335     }
8336
8337     my $bootorder = {};
8338
8339     PVE::QemuConfig->foreach_volume($conf, sub {
8340         my ($ds, $drive) = @_;
8341
8342         if (drive_is_cdrom ($drive, 1)) {
8343             if ($bootindex_hash->{d}) {
8344                 $bootorder->{$ds} = $bootindex_hash->{d};
8345                 $bootindex_hash->{d} += 1;
8346             }
8347         } elsif ($bootindex_hash->{c}) {
8348             $bootorder->{$ds} = $bootindex_hash->{c}
8349                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8350             $bootindex_hash->{c} += 1;
8351         }
8352     });
8353
8354     if ($bootindex_hash->{n}) {
8355         for (my $i = 0; $i < $MAX_NETS; $i++) {
8356             my $netname = "net$i";
8357             next if !$conf->{$netname};
8358             $bootorder->{$netname} = $bootindex_hash->{n};
8359             $bootindex_hash->{n} += 1;
8360         }
8361     }
8362
8363     return $bootorder;
8364 }
8365
8366 # Generate default device list for 'boot: order=' property. Matches legacy
8367 # default boot order, but with explicit device names. This is important, since
8368 # the fallback for when neither 'order' nor the old format is specified relies
8369 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8370 sub get_default_bootdevices {
8371     my ($conf) = @_;
8372
8373     my @ret = ();
8374
8375     # harddisk
8376     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8377     push @ret, $first if $first;
8378
8379     # cdrom
8380     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8381     push @ret, $first if $first;
8382
8383     # network
8384     for (my $i = 0; $i < $MAX_NETS; $i++) {
8385         my $netname = "net$i";
8386         next if !$conf->{$netname};
8387         push @ret, $netname;
8388         last;
8389     }
8390
8391     return \@ret;
8392 }
8393
8394 sub device_bootorder {
8395     my ($conf) = @_;
8396
8397     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8398
8399     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8400
8401     my $bootorder = {};
8402     if (!defined($boot) || $boot->{legacy}) {
8403         $bootorder = bootorder_from_legacy($conf, $boot);
8404     } elsif ($boot->{order}) {
8405         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8406         for my $dev (PVE::Tools::split_list($boot->{order})) {
8407             $bootorder->{$dev} = $i++;
8408         }
8409     }
8410
8411     return $bootorder;
8412 }
8413
8414 sub register_qmeventd_handle {
8415     my ($vmid) = @_;
8416
8417     my $fh;
8418     my $peer = "/var/run/qmeventd.sock";
8419     my $count = 0;
8420
8421     for (;;) {
8422         $count++;
8423         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8424         last if $fh;
8425         if ($! != EINTR && $! != EAGAIN) {
8426             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8427         }
8428         if ($count > 4) {
8429             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8430               . "after $count retries\n";
8431         }
8432         usleep(25000);
8433     }
8434
8435     # send handshake to mark VM as backing up
8436     print $fh to_json({vzdump => {vmid => "$vmid"}});
8437
8438     # return handle to be closed later when inhibit is no longer required
8439     return $fh;
8440 }
8441
8442 # bash completion helper
8443
8444 sub complete_backup_archives {
8445     my ($cmdname, $pname, $cvalue) = @_;
8446
8447     my $cfg = PVE::Storage::config();
8448
8449     my $storeid;
8450
8451     if ($cvalue =~ m/^([^:]+):/) {
8452         $storeid = $1;
8453     }
8454
8455     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8456
8457     my $res = [];
8458     foreach my $id (keys %$data) {
8459         foreach my $item (@{$data->{$id}}) {
8460             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8461             push @$res, $item->{volid} if defined($item->{volid});
8462         }
8463     }
8464
8465     return $res;
8466 }
8467
8468 my $complete_vmid_full = sub {
8469     my ($running) = @_;
8470
8471     my $idlist = vmstatus();
8472
8473     my $res = [];
8474
8475     foreach my $id (keys %$idlist) {
8476         my $d = $idlist->{$id};
8477         if (defined($running)) {
8478             next if $d->{template};
8479             next if $running && $d->{status} ne 'running';
8480             next if !$running && $d->{status} eq 'running';
8481         }
8482         push @$res, $id;
8483
8484     }
8485     return $res;
8486 };
8487
8488 sub complete_vmid {
8489     return &$complete_vmid_full();
8490 }
8491
8492 sub complete_vmid_stopped {
8493     return &$complete_vmid_full(0);
8494 }
8495
8496 sub complete_vmid_running {
8497     return &$complete_vmid_full(1);
8498 }
8499
8500 sub complete_storage {
8501
8502     my $cfg = PVE::Storage::config();
8503     my $ids = $cfg->{ids};
8504
8505     my $res = [];
8506     foreach my $sid (keys %$ids) {
8507         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8508         next if !$ids->{$sid}->{content}->{images};
8509         push @$res, $sid;
8510     }
8511
8512     return $res;
8513 }
8514
8515 sub complete_migration_storage {
8516     my ($cmd, $param, $current_value, $all_args) = @_;
8517
8518     my $targetnode = @$all_args[1];
8519
8520     my $cfg = PVE::Storage::config();
8521     my $ids = $cfg->{ids};
8522
8523     my $res = [];
8524     foreach my $sid (keys %$ids) {
8525         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8526         next if !$ids->{$sid}->{content}->{images};
8527         push @$res, $sid;
8528     }
8529
8530     return $res;
8531 }
8532
8533 sub vm_is_paused {
8534     my ($vmid, $include_suspended) = @_;
8535     my $qmpstatus = eval {
8536         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8537         mon_cmd($vmid, "query-status");
8538     };
8539     warn "$@\n" if $@;
8540     return $qmpstatus && (
8541         $qmpstatus->{status} eq "paused" ||
8542         $qmpstatus->{status} eq "prelaunch" ||
8543         ($include_suspended && $qmpstatus->{status} eq "suspended")
8544     );
8545 }
8546
8547 sub check_volume_storage_type {
8548     my ($storecfg, $vol) = @_;
8549
8550     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8551     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8552     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8553
8554     die "storage '$storeid' does not support content-type '$vtype'\n"
8555         if !$scfg->{content}->{$vtype};
8556
8557     return 1;
8558 }
8559
8560 sub add_nets_bridge_fdb {
8561     my ($conf, $vmid) = @_;
8562
8563     for my $opt (keys %$conf) {
8564         next if $opt !~ m/^net(\d+)$/;
8565         my $iface = "tap${vmid}i$1";
8566         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8567         my $net = parse_net($conf->{$opt}, 1) or next;
8568
8569         my $mac = $net->{macaddr};
8570         if (!$mac) {
8571             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8572                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8573             next;
8574         }
8575
8576         my $bridge = $net->{bridge};
8577         if (!$bridge) {
8578             log_warn("Interface '$iface' not attached to any bridge.");
8579             next;
8580         }
8581         if ($have_sdn) {
8582             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8583         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8584             PVE::Network::add_bridge_fdb($iface, $mac);
8585         }
8586     }
8587 }
8588
8589 sub del_nets_bridge_fdb {
8590     my ($conf, $vmid) = @_;
8591
8592     for my $opt (keys %$conf) {
8593         next if $opt !~ m/^net(\d+)$/;
8594         my $iface = "tap${vmid}i$1";
8595
8596         my $net = parse_net($conf->{$opt}) or next;
8597         my $mac = $net->{macaddr} or next;
8598
8599         my $bridge = $net->{bridge};
8600         if ($have_sdn) {
8601             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8602         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8603             PVE::Network::del_bridge_fdb($iface, $mac);
8604         }
8605     }
8606 }
8607
8608 sub create_ifaces_ipams_ips {
8609     my ($conf, $vmid) = @_;
8610
8611     return if !$have_sdn;
8612
8613     foreach my $opt (keys %$conf) {
8614         if ($opt =~ m/^net(\d+)$/) {
8615             my $value = $conf->{$opt};
8616             my $net = PVE::QemuServer::parse_net($value);
8617             eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8618             warn $@ if $@;
8619         }
8620     }
8621 }
8622
8623 sub delete_ifaces_ipams_ips {
8624     my ($conf, $vmid) = @_;
8625
8626     return if !$have_sdn;
8627
8628     foreach my $opt (keys %$conf) {
8629         if ($opt =~ m/^net(\d+)$/) {
8630             my $net = PVE::QemuServer::parse_net($conf->{$opt});
8631             eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8632             warn $@ if $@;
8633         }
8634     }
8635 }
8636
8637 1;