PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory;
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::USB;
  62
  63 my $have_sdn;
  64 eval {
  65     require PVE::Network::SDN::Zones;
  66     $have_sdn = 1;
  67 };
  68
  69 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  70 my $OVMF = {
  71     x86_64 => {
  72         '4m-no-smm' => [
  73             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  74             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  75         ],
  76         '4m-no-smm-ms' => [
  77             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  78             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  79         ],
  80         '4m' => [
  81             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  82             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  83         ],
  84         '4m-ms' => [
  85             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  86             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  87         ],
  88         default => [
  89             "$EDK2_FW_BASE/OVMF_CODE.fd",
  90             "$EDK2_FW_BASE/OVMF_VARS.fd",
  91         ],
  92     },
  93     aarch64 => {
  94         default => [
  95             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  96             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  97         ],
  98     },
  99 };
 100
 101 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 102
 103 # Note about locking: we use flock on the config file protect against concurent actions.
 104 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 105 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 106 # But you can ignore this kind of lock with the --skiplock flag.
 107
 108 cfs_register_file(
 109     '/qemu-server/',
 110     \&parse_vm_config,
 111     \&write_vm_config
 112 );
 113
 114 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 115     description => "Some command save/restore state from this location.",
 116     type => 'string',
 117     maxLength => 128,
 118     optional => 1,
 119 });
 120
 121 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 122         description => "Specifies the QEMU machine type.",
 123         type => 'string',
 124         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 125         maxLength => 40,
 126         optional => 1,
 127 });
 128
 129 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 130 my $nodename_cache;
 131 sub nodename {
 132     $nodename_cache //= PVE::INotify::nodename();
 133     return $nodename_cache;
 134 }
 135
 136 my $watchdog_fmt = {
 137     model => {
 138         default_key => 1,
 139         type => 'string',
 140         enum => [qw(i6300esb ib700)],
 141         description => "Watchdog type to emulate.",
 142         default => 'i6300esb',
 143         optional => 1,
 144     },
 145     action => {
 146         type => 'string',
 147         enum => [qw(reset shutdown poweroff pause debug none)],
 148         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 149         optional => 1,
 150     },
 151 };
 152 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 153
 154 my $agent_fmt = {
 155     enabled => {
 156         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 157         type => 'boolean',
 158         default => 0,
 159         default_key => 1,
 160     },
 161     fstrim_cloned_disks => {
 162         description => "Run fstrim after moving a disk or migrating the VM.",
 163         type => 'boolean',
 164         optional => 1,
 165         default => 0,
 166     },
 167     'freeze-fs-on-backup' => {
 168         description => "Freeze/thaw guest filesystems on backup for consistency.",
 169         type => 'boolean',
 170         optional => 1,
 171         default => 1,
 172     },
 173     type => {
 174         description => "Select the agent type",
 175         type => 'string',
 176         default => 'virtio',
 177         optional => 1,
 178         enum => [qw(virtio isa)],
 179     },
 180 };
 181
 182 my $vga_fmt = {
 183     type => {
 184         description => "Select the VGA type.",
 185         type => 'string',
 186         default => 'std',
 187         optional => 1,
 188         default_key => 1,
 189         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 190     },
 191     memory => {
 192         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 193         type => 'integer',
 194         optional => 1,
 195         minimum => 4,
 196         maximum => 512,
 197     },
 198 };
 199
 200 my $ivshmem_fmt = {
 201     size => {
 202         type => 'integer',
 203         minimum => 1,
 204         description => "The size of the file in MB.",
 205     },
 206     name => {
 207         type => 'string',
 208         pattern => '[a-zA-Z0-9\-]+',
 209         optional => 1,
 210         format_description => 'string',
 211         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 212     },
 213 };
 214
 215 my $audio_fmt = {
 216     device => {
 217         type => 'string',
 218         enum => [qw(ich9-intel-hda intel-hda AC97)],
 219         description =>  "Configure an audio device."
 220     },
 221     driver =>  {
 222         type => 'string',
 223         enum => ['spice', 'none'],
 224         default => 'spice',
 225         optional => 1,
 226         description => "Driver backend for the audio device."
 227     },
 228 };
 229
 230 my $spice_enhancements_fmt = {
 231     foldersharing => {
 232         type => 'boolean',
 233         optional => 1,
 234         default => '0',
 235         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 236     },
 237     videostreaming =>  {
 238         type => 'string',
 239         enum => ['off', 'all', 'filter'],
 240         default => 'off',
 241         optional => 1,
 242         description => "Enable video streaming. Uses compression for detected video streams."
 243     },
 244 };
 245
 246 my $rng_fmt = {
 247     source => {
 248         type => 'string',
 249         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 250         default_key => 1,
 251         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 252             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 253             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 254             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 255             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 256             ." a hardware RNG from the host.",
 257     },
 258     max_bytes => {
 259         type => 'integer',
 260         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 261             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 262             ." `0` to disable limiting (potentially dangerous!).",
 263         optional => 1,
 264
 265         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 266         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 267         # reading from /dev/urandom
 268         default => 1024,
 269     },
 270     period => {
 271         type => 'integer',
 272         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 273             ." the guest to retrieve another 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $meta_info_fmt = {
 280     'ctime' => {
 281         type => 'integer',
 282         description => "The guest creation timestamp as UNIX epoch time",
 283         minimum => 0,
 284         optional => 1,
 285     },
 286     'creation-qemu' => {
 287         type => 'string',
 288         description => "The QEMU (machine) version from the time this VM was created.",
 289         pattern => '\d+(\.\d+)+',
 290         optional => 1,
 291     },
 292 };
 293
 294 my $confdesc = {
 295     onboot => {
 296         optional => 1,
 297         type => 'boolean',
 298         description => "Specifies whether a VM will be started during system bootup.",
 299         default => 0,
 300     },
 301     autostart => {
 302         optional => 1,
 303         type => 'boolean',
 304         description => "Automatic restart after crash (currently ignored).",
 305         default => 0,
 306     },
 307     hotplug => {
 308         optional => 1,
 309         type => 'string', format => 'pve-hotplug-features',
 310         description => "Selectively enable hotplug features. This is a comma separated list of"
 311             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 312             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 313             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 314             ." windows > 7.",
 315         default => 'network,disk,usb',
 316     },
 317     reboot => {
 318         optional => 1,
 319         type => 'boolean',
 320         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 321         default => 1,
 322     },
 323     lock => {
 324         optional => 1,
 325         type => 'string',
 326         description => "Lock/unlock the VM.",
 327         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 328     },
 329     cpulimit => {
 330         optional => 1,
 331         type => 'number',
 332         description => "Limit of CPU usage.",
 333         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 334             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 335         minimum => 0,
 336         maximum => 128,
 337         default => 0,
 338     },
 339     cpuunits => {
 340         optional => 1,
 341         type => 'integer',
 342         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 343         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 344             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 345             ." weights of all the other running VMs.",
 346         minimum => 1,
 347         maximum => 262144,
 348         default => 'cgroup v1: 1024, cgroup v2: 100',
 349     },
 350     memory => {
 351         optional => 1,
 352         type => 'integer',
 353         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 354             ." you use the balloon device.",
 355         minimum => 16,
 356         default => 512,
 357     },
 358     balloon => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 362         minimum => 0,
 363     },
 364     shares => {
 365         optional => 1,
 366         type => 'integer',
 367         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 368             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 369             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 370         minimum => 0,
 371         maximum => 50000,
 372         default => 1000,
 373     },
 374     keyboard => {
 375         optional => 1,
 376         type => 'string',
 377         description => "Keyboard layout for VNC server. This option is generally not required and"
 378         ." is often better handled from within the guest OS.",
 379         enum => PVE::Tools::kvmkeymaplist(),
 380         default => undef,
 381     },
 382     name => {
 383         optional => 1,
 384         type => 'string', format => 'dns-name',
 385         description => "Set a name for the VM. Only used on the configuration web interface.",
 386     },
 387     scsihw => {
 388         optional => 1,
 389         type => 'string',
 390         description => "SCSI controller model",
 391         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 392         default => 'lsi',
 393     },
 394     description => {
 395         optional => 1,
 396         type => 'string',
 397         description => "Description for the VM. Shown in the web-interface VM's summary."
 398             ." This is saved as comment inside the configuration file.",
 399         maxLength => 1024 * 8,
 400     },
 401     ostype => {
 402         optional => 1,
 403         type => 'string',
 404         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 405         description => "Specify guest operating system.",
 406         verbose_description => <<EODESC,
 407 Specify guest operating system. This is used to enable special
 408 optimization/features for specific operating systems:
 409
 410 [horizontal]
 411 other;; unspecified OS
 412 wxp;; Microsoft Windows XP
 413 w2k;; Microsoft Windows 2000
 414 w2k3;; Microsoft Windows 2003
 415 w2k8;; Microsoft Windows 2008
 416 wvista;; Microsoft Windows Vista
 417 win7;; Microsoft Windows 7
 418 win8;; Microsoft Windows 8/2012/2012r2
 419 win10;; Microsoft Windows 10/2016/2019
 420 win11;; Microsoft Windows 11/2022
 421 l24;; Linux 2.4 Kernel
 422 l26;; Linux 2.6 - 6.X Kernel
 423 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 424 EODESC
 425     },
 426     boot => {
 427         optional => 1,
 428         type => 'string', format => 'pve-qm-boot',
 429         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 430             ." key or 'legacy=' is deprecated.",
 431     },
 432     bootdisk => {
 433         optional => 1,
 434         type => 'string', format => 'pve-qm-bootdisk',
 435         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 436         pattern => '(ide|sata|scsi|virtio)\d+',
 437     },
 438     smp => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPUs. Please use option -sockets instead.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     sockets => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of CPU sockets.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     cores => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of cores per socket.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     numa => {
 460         optional => 1,
 461         type => 'boolean',
 462         description => "Enable/disable NUMA.",
 463         default => 0,
 464     },
 465     hugepages => {
 466         optional => 1,
 467         type => 'string',
 468         description => "Enable/disable hugepages memory.",
 469         enum => [qw(any 2 1024)],
 470     },
 471     keephugepages => {
 472         optional => 1,
 473         type => 'boolean',
 474         default => 0,
 475         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 476             ." after VM shutdown and can be used for subsequent starts.",
 477     },
 478     vcpus => {
 479         optional => 1,
 480         type => 'integer',
 481         description => "Number of hotplugged vcpus.",
 482         minimum => 1,
 483         default => 0,
 484     },
 485     acpi => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Enable/disable ACPI.",
 489         default => 1,
 490     },
 491     agent => {
 492         optional => 1,
 493         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 494         type => 'string',
 495         format => $agent_fmt,
 496     },
 497     kvm => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Enable/disable KVM hardware virtualization.",
 501         default => 1,
 502     },
 503     tdf => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable time drift fix.",
 507         default => 0,
 508     },
 509     localtime => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 513             ." the `ostype` indicates a Microsoft Windows OS.",
 514     },
 515     freeze => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 519     },
 520     vga => {
 521         optional => 1,
 522         type => 'string', format => $vga_fmt,
 523         description => "Configure the VGA hardware.",
 524         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 525             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 526             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 527             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 528             ." display server. For win* OS you can select how many independent displays you want,"
 529             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 530             ." using a serial device as terminal.",
 531     },
 532     watchdog => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-watchdog',
 535         description => "Create a virtual hardware watchdog device.",
 536         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 537             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 538             ." else the watchdog will reset the guest (or execute the respective action specified)",
 539     },
 540     startdate => {
 541         optional => 1,
 542         type => 'string',
 543         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 544         description => "Set the initial date of the real time clock. Valid format for date are:"
 545             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 546         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 547         default => 'now',
 548     },
 549     startup =>  get_standard_option('pve-startup-order'),
 550     template => {
 551         optional => 1,
 552         type => 'boolean',
 553         description => "Enable/disable Template.",
 554         default => 0,
 555     },
 556     args => {
 557         optional => 1,
 558         type => 'string',
 559         description => "Arbitrary arguments passed to kvm.",
 560         verbose_description => <<EODESCR,
 561 Arbitrary arguments passed to kvm, for example:
 562
 563 args: -no-reboot -smbios 'type=0,vendor=FOO'
 564
 565 NOTE: this option is for experts only.
 566 EODESCR
 567     },
 568     tablet => {
 569         optional => 1,
 570         type => 'boolean',
 571         default => 1,
 572         description => "Enable/disable the USB tablet device.",
 573         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 574             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 575             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 576             ." may consider disabling this to save some context switches. This is turned off by"
 577             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 578     },
 579     migrate_speed => {
 580         optional => 1,
 581         type => 'integer',
 582         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 583         minimum => 0,
 584         default => 0,
 585     },
 586     migrate_downtime => {
 587         optional => 1,
 588         type => 'number',
 589         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 590         minimum => 0,
 591         default => 0.1,
 592     },
 593     cdrom => {
 594         optional => 1,
 595         type => 'string', format => 'pve-qm-ide',
 596         typetext => '<volume>',
 597         description => "This is an alias for option -ide2",
 598     },
 599     cpu => {
 600         optional => 1,
 601         description => "Emulated CPU type.",
 602         type => 'string',
 603         format => 'pve-vm-cpu-conf',
 604     },
 605     parent => get_standard_option('pve-snapshot-name', {
 606         optional => 1,
 607         description => "Parent snapshot name. This is used internally, and should not be modified.",
 608     }),
 609     snaptime => {
 610         optional => 1,
 611         description => "Timestamp for snapshots.",
 612         type => 'integer',
 613         minimum => 0,
 614     },
 615     vmstate => {
 616         optional => 1,
 617         type => 'string', format => 'pve-volume-id',
 618         description => "Reference to a volume which stores the VM state. This is used internally"
 619             ." for snapshots.",
 620     },
 621     vmstatestorage => get_standard_option('pve-storage-id', {
 622         description => "Default storage for VM state volumes/files.",
 623         optional => 1,
 624     }),
 625     runningmachine => get_standard_option('pve-qemu-machine', {
 626         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 627             ." for snapshots.",
 628     }),
 629     runningcpu => {
 630         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 631             ." internally for snapshots.",
 632         optional => 1,
 633         type => 'string',
 634         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 635         format_description => 'QEMU -cpu parameter'
 636     },
 637     machine => get_standard_option('pve-qemu-machine'),
 638     arch => {
 639         description => "Virtual processor architecture. Defaults to the host.",
 640         optional => 1,
 641         type => 'string',
 642         enum => [qw(x86_64 aarch64)],
 643     },
 644     smbios1 => {
 645         description => "Specify SMBIOS type 1 fields.",
 646         type => 'string', format => 'pve-qm-smbios1',
 647         maxLength => 512,
 648         optional => 1,
 649     },
 650     protection => {
 651         optional => 1,
 652         type => 'boolean',
 653         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 654             ." remove disk operations.",
 655         default => 0,
 656     },
 657     bios => {
 658         optional => 1,
 659         type => 'string',
 660         enum => [ qw(seabios ovmf) ],
 661         description => "Select BIOS implementation.",
 662         default => 'seabios',
 663     },
 664     vmgenid => {
 665         type => 'string',
 666         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 667         format_description => 'UUID',
 668         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 669             ." to disable explicitly.",
 670         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 671             ." value identifier to the guest OS. This allows to notify the guest operating system"
 672             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 673             ." execution or creation from a template). The guest operating system notices the"
 674             ." change, and is then able to react as appropriate by marking its copies of"
 675             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 676             ."Note that auto-creation only works when done through API/CLI create or update methods"
 677             .", but not when manually editing the config file.",
 678         default => "1 (autogenerated)",
 679         optional => 1,
 680     },
 681     hookscript => {
 682         type => 'string',
 683         format => 'pve-volume-id',
 684         optional => 1,
 685         description => "Script that will be executed during various steps in the vms lifetime.",
 686     },
 687     ivshmem => {
 688         type => 'string',
 689         format => $ivshmem_fmt,
 690         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 691             ." the host.",
 692         optional => 1,
 693     },
 694     audio0 => {
 695         type => 'string',
 696         format => $audio_fmt,
 697         description => "Configure a audio device, useful in combination with QXL/Spice.",
 698         optional => 1
 699     },
 700     spice_enhancements => {
 701         type => 'string',
 702         format => $spice_enhancements_fmt,
 703         description => "Configure additional enhancements for SPICE.",
 704         optional => 1
 705     },
 706     tags => {
 707         type => 'string', format => 'pve-tag-list',
 708         description => 'Tags of the VM. This is only meta information.',
 709         optional => 1,
 710     },
 711     rng0 => {
 712         type => 'string',
 713         format => $rng_fmt,
 714         description => "Configure a VirtIO-based Random Number Generator.",
 715         optional => 1,
 716     },
 717     meta => {
 718         type => 'string',
 719         format => $meta_info_fmt,
 720         description => "Some (read-only) meta-information about this guest.",
 721         optional => 1,
 722     },
 723     affinity => {
 724         type => 'string', format => 'pve-cpuset',
 725         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 726         optional => 1,
 727     },
 728 };
 729
 730 my $cicustom_fmt = {
 731     meta => {
 732         type => 'string',
 733         optional => 1,
 734         description => 'Specify a custom file containing all meta data passed to the VM via"
 735             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 736         format => 'pve-volume-id',
 737         format_description => 'volume',
 738     },
 739     network => {
 740         type => 'string',
 741         optional => 1,
 742         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 743         format => 'pve-volume-id',
 744         format_description => 'volume',
 745     },
 746     user => {
 747         type => 'string',
 748         optional => 1,
 749         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 750         format => 'pve-volume-id',
 751         format_description => 'volume',
 752     },
 753     vendor => {
 754         type => 'string',
 755         optional => 1,
 756         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 757         format => 'pve-volume-id',
 758         format_description => 'volume',
 759     },
 760 };
 761 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 762
 763 my $confdesc_cloudinit = {
 764     citype => {
 765         optional => 1,
 766         type => 'string',
 767         description => 'Specifies the cloud-init configuration format. The default depends on the'
 768             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 769             .' and `configdrive2` for windows.',
 770         enum => ['configdrive2', 'nocloud', 'opennebula'],
 771     },
 772     ciuser => {
 773         optional => 1,
 774         type => 'string',
 775         description => "cloud-init: User name to change ssh keys and password for instead of the"
 776             ." image's configured default user.",
 777     },
 778     cipassword => {
 779         optional => 1,
 780         type => 'string',
 781         description => 'cloud-init: Password to assign the user. Using this is generally not'
 782             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 783             .' support hashed passwords.',
 784     },
 785     ciupgrade => {
 786         optional => 1,
 787         type => 'boolean',
 788         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 789         default => 1,
 790     },
 791     cicustom => {
 792         optional => 1,
 793         type => 'string',
 794         description => 'cloud-init: Specify custom files to replace the automatically generated'
 795             .' ones at start.',
 796         format => 'pve-qm-cicustom',
 797     },
 798     searchdomain => {
 799         optional => 1,
 800         type => 'string',
 801         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 802             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 803             .' are set.',
 804     },
 805     nameserver => {
 806         optional => 1,
 807         type => 'string', format => 'address-list',
 808         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 809             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 810             .' are set.',
 811     },
 812     sshkeys => {
 813         optional => 1,
 814         type => 'string',
 815         format => 'urlencoded',
 816         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 817     },
 818 };
 819
 820 # what about other qemu settings ?
 821 #cpu => 'string',
 822 #machine => 'string',
 823 #fda => 'file',
 824 #fdb => 'file',
 825 #mtdblock => 'file',
 826 #sd => 'file',
 827 #pflash => 'file',
 828 #snapshot => 'bool',
 829 #bootp => 'file',
 830 ##tftp => 'dir',
 831 ##smb => 'dir',
 832 #kernel => 'file',
 833 #append => 'string',
 834 #initrd => 'file',
 835 ##soundhw => 'string',
 836
 837 while (my ($k, $v) = each %$confdesc) {
 838     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 839 }
 840
 841 my $MAX_NETS = 32;
 842 my $MAX_SERIAL_PORTS = 4;
 843 my $MAX_PARALLEL_PORTS = 3;
 844 my $MAX_NUMA = 8;
 845
 846 my $numa_fmt = {
 847     cpus => {
 848         type => "string",
 849         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 850         description => "CPUs accessing this NUMA node.",
 851         format_description => "id[-id];...",
 852     },
 853     memory => {
 854         type => "number",
 855         description => "Amount of memory this NUMA node provides.",
 856         optional => 1,
 857     },
 858     hostnodes => {
 859         type => "string",
 860         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 861         description => "Host NUMA nodes to use.",
 862         format_description => "id[-id];...",
 863         optional => 1,
 864     },
 865     policy => {
 866         type => 'string',
 867         enum => [qw(preferred bind interleave)],
 868         description => "NUMA allocation policy.",
 869         optional => 1,
 870     },
 871 };
 872 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 873 my $numadesc = {
 874     optional => 1,
 875     type => 'string', format => $numa_fmt,
 876     description => "NUMA topology.",
 877 };
 878 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 879
 880 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 881     $confdesc->{"numa$i"} = $numadesc;
 882 }
 883
 884 my $nic_model_list = [
 885     'e1000',
 886     'e1000-82540em',
 887     'e1000-82544gc',
 888     'e1000-82545em',
 889     'e1000e',
 890     'i82551',
 891     'i82557b',
 892     'i82559er',
 893     'ne2k_isa',
 894     'ne2k_pci',
 895     'pcnet',
 896     'rtl8139',
 897     'virtio',
 898     'vmxnet3',
 899 ];
 900 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 901
 902 my $net_fmt_bridge_descr = <<__EOD__;
 903 Bridge to attach the network device to. The Proxmox VE standard bridge
 904 is called 'vmbr0'.
 905
 906 If you do not specify a bridge, we create a kvm user (NATed) network
 907 device, which provides DHCP and DNS services. The following addresses
 908 are used:
 909
 910  10.0.2.2   Gateway
 911  10.0.2.3   DNS Server
 912  10.0.2.4   SMB Server
 913
 914 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 915 __EOD__
 916
 917 my $net_fmt = {
 918     macaddr  => get_standard_option('mac-addr', {
 919         description => "MAC address. That address must be unique withing your network. This is"
 920             ." automatically generated if not specified.",
 921     }),
 922     model => {
 923         type => 'string',
 924         description => "Network Card Model. The 'virtio' model provides the best performance with"
 925             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 926             ." best to use 'e1000'.",
 927         enum => $nic_model_list,
 928         default_key => 1,
 929     },
 930     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 931     bridge => get_standard_option('pve-bridge-id', {
 932         description => $net_fmt_bridge_descr,
 933         optional => 1,
 934     }),
 935     queues => {
 936         type => 'integer',
 937         minimum => 0, maximum => 64,
 938         description => 'Number of packet queues to be used on the device.',
 939         optional => 1,
 940     },
 941     rate => {
 942         type => 'number',
 943         minimum => 0,
 944         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 945         optional => 1,
 946     },
 947     tag => {
 948         type => 'integer',
 949         minimum => 1, maximum => 4094,
 950         description => 'VLAN tag to apply to packets on this interface.',
 951         optional => 1,
 952     },
 953     trunks => {
 954         type => 'string',
 955         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 956         description => 'VLAN trunks to pass through this interface.',
 957         format_description => 'vlanid[;vlanid...]',
 958         optional => 1,
 959     },
 960     firewall => {
 961         type => 'boolean',
 962         description => 'Whether this interface should be protected by the firewall.',
 963         optional => 1,
 964     },
 965     link_down => {
 966         type => 'boolean',
 967         description => 'Whether this interface should be disconnected (like pulling the plug).',
 968         optional => 1,
 969     },
 970     mtu => {
 971         type => 'integer',
 972         minimum => 1, maximum => 65520,
 973         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 974         optional => 1,
 975     },
 976 };
 977
 978 my $netdesc = {
 979     optional => 1,
 980     type => 'string', format => $net_fmt,
 981     description => "Specify network devices.",
 982 };
 983
 984 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 985
 986 my $ipconfig_fmt = {
 987     ip => {
 988         type => 'string',
 989         format => 'pve-ipv4-config',
 990         format_description => 'IPv4Format/CIDR',
 991         description => 'IPv4 address in CIDR format.',
 992         optional => 1,
 993         default => 'dhcp',
 994     },
 995     gw => {
 996         type => 'string',
 997         format => 'ipv4',
 998         format_description => 'GatewayIPv4',
 999         description => 'Default gateway for IPv4 traffic.',
1000         optional => 1,
1001         requires => 'ip',
1002     },
1003     ip6 => {
1004         type => 'string',
1005         format => 'pve-ipv6-config',
1006         format_description => 'IPv6Format/CIDR',
1007         description => 'IPv6 address in CIDR format.',
1008         optional => 1,
1009         default => 'dhcp',
1010     },
1011     gw6 => {
1012         type => 'string',
1013         format => 'ipv6',
1014         format_description => 'GatewayIPv6',
1015         description => 'Default gateway for IPv6 traffic.',
1016         optional => 1,
1017         requires => 'ip6',
1018     },
1019 };
1020 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1021 my $ipconfigdesc = {
1022     optional => 1,
1023     type => 'string', format => 'pve-qm-ipconfig',
1024     description => <<'EODESCR',
1025 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1026
1027 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1028
1029 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1030 gateway should be provided.
1031 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1032 cloud-init 19.4 or newer.
1033
1034 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1035 dhcp on IPv4.
1036 EODESCR
1037 };
1038 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1039
1040 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1041     $confdesc->{"net$i"} = $netdesc;
1042     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1043 }
1044
1045 foreach my $key (keys %$confdesc_cloudinit) {
1046     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1047 }
1048
1049 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1050 sub pve_verify_cpuset {
1051     my ($set_text, $noerr) = @_;
1052
1053     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1054
1055     if ($@) {
1056         return if $noerr;
1057         die "unable to parse cpuset option\n";
1058     }
1059
1060     return PVE::CpuSet->new($members)->short_string();
1061 }
1062
1063 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1064 sub verify_volume_id_or_qm_path {
1065     my ($volid, $noerr) = @_;
1066
1067     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1068
1069     return verify_volume_id_or_absolute_path($volid, $noerr);
1070 }
1071
1072 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1073 sub verify_volume_id_or_absolute_path {
1074     my ($volid, $noerr) = @_;
1075
1076     return $volid if $volid =~ m|^/|;
1077
1078     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1079     if ($@) {
1080         return if $noerr;
1081         die $@;
1082     }
1083     return $volid;
1084 }
1085
1086 my $serialdesc = {
1087         optional => 1,
1088         type => 'string',
1089         pattern => '(/dev/.+|socket)',
1090         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1091         verbose_description =>  <<EODESCR,
1092 Create a serial device inside the VM (n is 0 to 3), and pass through a
1093 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1094 host side (use 'qm terminal' to open a terminal connection).
1095
1096 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1097 use with special care.
1098
1099 CAUTION: Experimental! User reported problems with this option.
1100 EODESCR
1101 };
1102
1103 my $paralleldesc= {
1104         optional => 1,
1105         type => 'string',
1106         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1107         description =>  "Map host parallel devices (n is 0 to 2).",
1108         verbose_description =>  <<EODESCR,
1109 Map host parallel devices (n is 0 to 2).
1110
1111 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1112 machines - use with special care.
1113
1114 CAUTION: Experimental! User reported problems with this option.
1115 EODESCR
1116 };
1117
1118 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1119     $confdesc->{"parallel$i"} = $paralleldesc;
1120 }
1121
1122 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1123     $confdesc->{"serial$i"} = $serialdesc;
1124 }
1125
1126 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1127     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1128 }
1129
1130 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1131     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1132 }
1133
1134 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1135     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1136 }
1137
1138 my $boot_fmt = {
1139     legacy => {
1140         optional => 1,
1141         default_key => 1,
1142         type => 'string',
1143         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1144                      . " Deprecated, use 'order=' instead.",
1145         pattern => '[acdn]{1,4}',
1146         format_description => "[acdn]{1,4}",
1147
1148         # note: this is also the fallback if boot: is not given at all
1149         default => 'cdn',
1150     },
1151     order => {
1152         optional => 1,
1153         type => 'string',
1154         format => 'pve-qm-bootdev-list',
1155         format_description => "device[;device...]",
1156         description => <<EODESC,
1157 The guest will attempt to boot from devices in the order they appear here.
1158
1159 Disks, optical drives and passed-through storage USB devices will be directly
1160 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1161 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1162
1163 Note that only devices in this list will be marked as bootable and thus loaded
1164 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1165 (e.g. software-raid), you need to specify all of them here.
1166
1167 Overrides the deprecated 'legacy=[acdn]*' value when given.
1168 EODESC
1169     },
1170 };
1171 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1172
1173 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1174 sub verify_bootdev {
1175     my ($dev, $noerr) = @_;
1176
1177     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1178     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1179
1180     my $check = sub {
1181         my ($base) = @_;
1182         return 0 if $dev !~ m/^$base\d+$/;
1183         return 0 if !$confdesc->{$dev};
1184         return 1;
1185     };
1186
1187     return $dev if $check->("net");
1188     return $dev if $check->("usb");
1189     return $dev if $check->("hostpci");
1190
1191     return if $noerr;
1192     die "invalid boot device '$dev'\n";
1193 }
1194
1195 sub print_bootorder {
1196     my ($devs) = @_;
1197     return "" if !@$devs;
1198     my $data = { order => join(';', @$devs) };
1199     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1200 }
1201
1202 my $kvm_api_version = 0;
1203
1204 sub kvm_version {
1205     return $kvm_api_version if $kvm_api_version;
1206
1207     open my $fh, '<', '/dev/kvm' or return;
1208
1209     # 0xae00 => KVM_GET_API_VERSION
1210     $kvm_api_version = ioctl($fh, 0xae00, 0);
1211     close($fh);
1212
1213     return $kvm_api_version;
1214 }
1215
1216 my $kvm_user_version = {};
1217 my $kvm_mtime = {};
1218
1219 sub kvm_user_version {
1220     my ($binary) = @_;
1221
1222     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1223     my $st = stat($binary);
1224
1225     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1226     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1227         $cachedmtime == $st->mtime;
1228
1229     $kvm_user_version->{$binary} = 'unknown';
1230     $kvm_mtime->{$binary} = $st->mtime;
1231
1232     my $code = sub {
1233         my $line = shift;
1234         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1235             $kvm_user_version->{$binary} = $2;
1236         }
1237     };
1238
1239     eval { run_command([$binary, '--version'], outfunc => $code); };
1240     warn $@ if $@;
1241
1242     return $kvm_user_version->{$binary};
1243
1244 }
1245 my sub extract_version {
1246     my ($machine_type, $version) = @_;
1247     $version = kvm_user_version() if !defined($version);
1248     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1249 }
1250
1251 sub kernel_has_vhost_net {
1252     return -c '/dev/vhost-net';
1253 }
1254
1255 sub option_exists {
1256     my $key = shift;
1257     return defined($confdesc->{$key});
1258 }
1259
1260 my $cdrom_path;
1261 sub get_cdrom_path {
1262
1263     return $cdrom_path if defined($cdrom_path);
1264
1265     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1266
1267     if (!defined($cdrom_path)) {
1268         log_warn("no physical CD-ROM available, ignoring");
1269         $cdrom_path = '';
1270     }
1271
1272     return $cdrom_path;
1273 }
1274
1275 sub get_iso_path {
1276     my ($storecfg, $vmid, $cdrom) = @_;
1277
1278     if ($cdrom eq 'cdrom') {
1279         return get_cdrom_path();
1280     } elsif ($cdrom eq 'none') {
1281         return '';
1282     } elsif ($cdrom =~ m|^/|) {
1283         return $cdrom;
1284     } else {
1285         return PVE::Storage::path($storecfg, $cdrom);
1286     }
1287 }
1288
1289 # try to convert old style file names to volume IDs
1290 sub filename_to_volume_id {
1291     my ($vmid, $file, $media) = @_;
1292
1293      if (!($file eq 'none' || $file eq 'cdrom' ||
1294           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1295
1296         return if $file =~ m|/|;
1297
1298         if ($media && $media eq 'cdrom') {
1299             $file = "local:iso/$file";
1300         } else {
1301             $file = "local:$vmid/$file";
1302         }
1303     }
1304
1305     return $file;
1306 }
1307
1308 sub verify_media_type {
1309     my ($opt, $vtype, $media) = @_;
1310
1311     return if !$media;
1312
1313     my $etype;
1314     if ($media eq 'disk') {
1315         $etype = 'images';
1316     } elsif ($media eq 'cdrom') {
1317         $etype = 'iso';
1318     } else {
1319         die "internal error";
1320     }
1321
1322     return if ($vtype eq $etype);
1323
1324     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1325 }
1326
1327 sub cleanup_drive_path {
1328     my ($opt, $storecfg, $drive) = @_;
1329
1330     # try to convert filesystem paths to volume IDs
1331
1332     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1333         ($drive->{file} !~ m|^/dev/.+|) &&
1334         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1335         ($drive->{file} !~ m/^\d+$/)) {
1336         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1337         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1338             if !$vtype;
1339         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1340         verify_media_type($opt, $vtype, $drive->{media});
1341         $drive->{file} = $volid;
1342     }
1343
1344     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1345 }
1346
1347 sub parse_hotplug_features {
1348     my ($data) = @_;
1349
1350     my $res = {};
1351
1352     return $res if $data eq '0';
1353
1354     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1355
1356     foreach my $feature (PVE::Tools::split_list($data)) {
1357         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1358             $res->{$1} = 1;
1359         } else {
1360             die "invalid hotplug feature '$feature'\n";
1361         }
1362     }
1363     return $res;
1364 }
1365
1366 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1367 sub pve_verify_hotplug_features {
1368     my ($value, $noerr) = @_;
1369
1370     return $value if parse_hotplug_features($value);
1371
1372     return if $noerr;
1373
1374     die "unable to parse hotplug option\n";
1375 }
1376
1377 sub scsi_inquiry {
1378     my($fh, $noerr) = @_;
1379
1380     my $SG_IO = 0x2285;
1381     my $SG_GET_VERSION_NUM = 0x2282;
1382
1383     my $versionbuf = "\x00" x 8;
1384     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1385     if (!$ret) {
1386         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1387         return;
1388     }
1389     my $version = unpack("I", $versionbuf);
1390     if ($version < 30000) {
1391         die "scsi generic interface too old\n"  if !$noerr;
1392         return;
1393     }
1394
1395     my $buf = "\x00" x 36;
1396     my $sensebuf = "\x00" x 8;
1397     my $cmd = pack("C x3 C x1", 0x12, 36);
1398
1399     # see /usr/include/scsi/sg.h
1400     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1401
1402     my $packet = pack(
1403         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1404     );
1405
1406     $ret = ioctl($fh, $SG_IO, $packet);
1407     if (!$ret) {
1408         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1409         return;
1410     }
1411
1412     my @res = unpack($sg_io_hdr_t, $packet);
1413     if ($res[17] || $res[18]) {
1414         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1415         return;
1416     }
1417
1418     my $res = {};
1419     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1420
1421     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1422     $res->{type} &= 0x1F;
1423
1424     return $res;
1425 }
1426
1427 sub path_is_scsi {
1428     my ($path) = @_;
1429
1430     my $fh = IO::File->new("+<$path") || return;
1431     my $res = scsi_inquiry($fh, 1);
1432     close($fh);
1433
1434     return $res;
1435 }
1436
1437 sub print_tabletdevice_full {
1438     my ($conf, $arch) = @_;
1439
1440     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1441
1442     # we use uhci for old VMs because tablet driver was buggy in older qemu
1443     my $usbbus;
1444     if ($q35 || $arch eq 'aarch64') {
1445         $usbbus = 'ehci';
1446     } else {
1447         $usbbus = 'uhci';
1448     }
1449
1450     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1451 }
1452
1453 sub print_keyboarddevice_full {
1454     my ($conf, $arch) = @_;
1455
1456     return if $arch ne 'aarch64';
1457
1458     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1459 }
1460
1461 my sub get_drive_id {
1462     my ($drive) = @_;
1463     return "$drive->{interface}$drive->{index}";
1464 }
1465
1466 sub print_drivedevice_full {
1467     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1468
1469     my $device = '';
1470     my $maxdev = 0;
1471
1472     my $drive_id = get_drive_id($drive);
1473     if ($drive->{interface} eq 'virtio') {
1474         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1475         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1476         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1477     } elsif ($drive->{interface} eq 'scsi') {
1478
1479         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1480         my $unit = $drive->{index} % $maxdev;
1481         my $devicetype = 'hd';
1482         my $path = '';
1483         if (drive_is_cdrom($drive)) {
1484             $devicetype = 'cd';
1485         } else {
1486             if ($drive->{file} =~ m|^/|) {
1487                 $path = $drive->{file};
1488                 if (my $info = path_is_scsi($path)) {
1489                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1490                         $devicetype = 'block';
1491                     } elsif ($info->{type} == 1) { # tape
1492                         $devicetype = 'generic';
1493                     }
1494                 }
1495             } else {
1496                  $path = PVE::Storage::path($storecfg, $drive->{file});
1497             }
1498
1499             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1500             my $version = extract_version($machine_type, kvm_user_version());
1501             if ($path =~ m/^iscsi\:\/\// &&
1502                !min_version($version, 4, 1)) {
1503                 $devicetype = 'generic';
1504             }
1505         }
1506
1507         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1508             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1509         } else {
1510             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1511                 .",lun=$drive->{index}";
1512         }
1513         $device .= ",drive=drive-$drive_id,id=$drive_id";
1514
1515         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1516             $device .= ",rotation_rate=1";
1517         }
1518         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1519
1520     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1521         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1522         my $controller = int($drive->{index} / $maxdev);
1523         my $unit = $drive->{index} % $maxdev;
1524         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1525
1526         $device = "ide-$devicetype";
1527         if ($drive->{interface} eq 'ide') {
1528             $device .= ",bus=ide.$controller,unit=$unit";
1529         } else {
1530             $device .= ",bus=ahci$controller.$unit";
1531         }
1532         $device .= ",drive=drive-$drive_id,id=$drive_id";
1533
1534         if ($devicetype eq 'hd') {
1535             if (my $model = $drive->{model}) {
1536                 $model = URI::Escape::uri_unescape($model);
1537                 $device .= ",model=$model";
1538             }
1539             if ($drive->{ssd}) {
1540                 $device .= ",rotation_rate=1";
1541             }
1542         }
1543         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1544     } elsif ($drive->{interface} eq 'usb') {
1545         die "implement me";
1546         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1547     } else {
1548         die "unsupported interface type";
1549     }
1550
1551     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1552
1553     if (my $serial = $drive->{serial}) {
1554         $serial = URI::Escape::uri_unescape($serial);
1555         $device .= ",serial=$serial";
1556     }
1557
1558
1559     return $device;
1560 }
1561
1562 sub get_initiator_name {
1563     my $initiator;
1564
1565     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1566     while (defined(my $line = <$fh>)) {
1567         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1568         $initiator = $1;
1569         last;
1570     }
1571     $fh->close();
1572
1573     return $initiator;
1574 }
1575
1576 my sub storage_allows_io_uring_default {
1577     my ($scfg, $cache_direct) = @_;
1578
1579     # io_uring with cache mode writeback or writethrough on krbd will hang...
1580     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1581
1582     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1583     # sometimes, just plain disable...
1584     return if $scfg && $scfg->{type} eq 'lvm';
1585
1586     # io_uring causes problems when used with CIFS since kernel 5.15
1587     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1588     return if $scfg && $scfg->{type} eq 'cifs';
1589
1590     return 1;
1591 }
1592
1593 my sub drive_uses_cache_direct {
1594     my ($drive, $scfg) = @_;
1595
1596     my $cache_direct = 0;
1597
1598     if (my $cache = $drive->{cache}) {
1599         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1600     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1601         $cache_direct = 1;
1602     }
1603
1604     return $cache_direct;
1605 }
1606
1607 sub print_drive_commandline_full {
1608     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1609
1610     my $path;
1611     my $volid = $drive->{file};
1612     my $format = $drive->{format};
1613     my $drive_id = get_drive_id($drive);
1614
1615     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1616     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1617
1618     if (drive_is_cdrom($drive)) {
1619         $path = get_iso_path($storecfg, $vmid, $volid);
1620         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1621     } else {
1622         if ($storeid) {
1623             $path = PVE::Storage::path($storecfg, $volid);
1624             $format //= qemu_img_format($scfg, $volname);
1625         } else {
1626             $path = $volid;
1627             $format //= "raw";
1628         }
1629    }
1630
1631    my $is_rbd = $path =~ m/^rbd:/;
1632
1633     my $opts = '';
1634     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1635     foreach my $o (@qemu_drive_options) {
1636         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1637     }
1638
1639     # snapshot only accepts on|off
1640     if (defined($drive->{snapshot})) {
1641         my $v = $drive->{snapshot} ? 'on' : 'off';
1642         $opts .= ",snapshot=$v";
1643     }
1644
1645     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1646         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1647     }
1648
1649     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1650         my ($dir, $qmpname) = @$type;
1651         if (my $v = $drive->{"mbps$dir"}) {
1652             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1653         }
1654         if (my $v = $drive->{"mbps${dir}_max"}) {
1655             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1656         }
1657         if (my $v = $drive->{"bps${dir}_max_length"}) {
1658             $opts .= ",throttling.bps$qmpname-max-length=$v";
1659         }
1660         if (my $v = $drive->{"iops${dir}"}) {
1661             $opts .= ",throttling.iops$qmpname=$v";
1662         }
1663         if (my $v = $drive->{"iops${dir}_max"}) {
1664             $opts .= ",throttling.iops$qmpname-max=$v";
1665         }
1666         if (my $v = $drive->{"iops${dir}_max_length"}) {
1667             $opts .= ",throttling.iops$qmpname-max-length=$v";
1668         }
1669     }
1670
1671     if ($pbs_name) {
1672         $format = "rbd" if $is_rbd;
1673         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1674             if !$format;
1675         $opts .= ",format=alloc-track,file.driver=$format";
1676     } elsif ($format) {
1677         $opts .= ",format=$format";
1678     }
1679
1680     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1681
1682     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1683
1684     if (!$drive->{aio}) {
1685         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1686             # io_uring supports all cache modes
1687             $opts .= ",aio=io_uring";
1688         } else {
1689             # aio native works only with O_DIRECT
1690             if($cache_direct) {
1691                 $opts .= ",aio=native";
1692             } else {
1693                 $opts .= ",aio=threads";
1694             }
1695         }
1696     }
1697
1698     if (!drive_is_cdrom($drive)) {
1699         my $detectzeroes;
1700         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1701             $detectzeroes = 'off';
1702         } elsif ($drive->{discard}) {
1703             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1704         } else {
1705             # This used to be our default with discard not being specified:
1706             $detectzeroes = 'on';
1707         }
1708
1709         # note: 'detect-zeroes' works per blockdev and we want it to persist
1710         # after the alloc-track is removed, so put it on 'file' directly
1711         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1712         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1713     }
1714
1715     if ($pbs_name) {
1716         $opts .= ",backing=$pbs_name";
1717         $opts .= ",auto-remove=on";
1718     }
1719
1720     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1721     my $file_param = "file";
1722     if ($pbs_name) {
1723         # non-rbd drivers require the underlying file to be a seperate block
1724         # node, so add a second .file indirection
1725         $file_param .= ".file" if !$is_rbd;
1726         $file_param .= ".filename";
1727     }
1728     my $pathinfo = $path ? "$file_param=$path," : '';
1729
1730     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1731 }
1732
1733 sub print_pbs_blockdev {
1734     my ($pbs_conf, $pbs_name) = @_;
1735     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1736     $blockdev .= ",repository=$pbs_conf->{repository}";
1737     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1738     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1739     $blockdev .= ",archive=$pbs_conf->{archive}";
1740     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1741     return $blockdev;
1742 }
1743
1744 sub print_netdevice_full {
1745     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1746
1747     my $device = $net->{model};
1748     if ($net->{model} eq 'virtio') {
1749          $device = 'virtio-net-pci';
1750      };
1751
1752     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1753     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1754     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1755         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1756         # and out of each queue plus one config interrupt and control vector queue
1757         my $vectors = $net->{queues} * 2 + 2;
1758         $tmpstr .= ",vectors=$vectors,mq=on";
1759         if (min_version($machine_version, 7, 1)) {
1760             $tmpstr .= ",packed=on";
1761         }
1762     }
1763
1764     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1765         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1766     }
1767
1768     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1769
1770     if (my $mtu = $net->{mtu}) {
1771         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1772             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1773             if ($mtu == 1) {
1774                  $mtu = $bridge_mtu;
1775             } elsif ($mtu < 576) {
1776                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1777             } elsif ($mtu > $bridge_mtu) {
1778                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1779             }
1780             $tmpstr .= ",host_mtu=$mtu";
1781         } else {
1782             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1783         }
1784     }
1785
1786     if ($use_old_bios_files) {
1787         my $romfile;
1788         if ($device eq 'virtio-net-pci') {
1789             $romfile = 'pxe-virtio.rom';
1790         } elsif ($device eq 'e1000') {
1791             $romfile = 'pxe-e1000.rom';
1792         } elsif ($device eq 'e1000e') {
1793             $romfile = 'pxe-e1000e.rom';
1794         } elsif ($device eq 'ne2k') {
1795             $romfile = 'pxe-ne2k_pci.rom';
1796         } elsif ($device eq 'pcnet') {
1797             $romfile = 'pxe-pcnet.rom';
1798         } elsif ($device eq 'rtl8139') {
1799             $romfile = 'pxe-rtl8139.rom';
1800         }
1801         $tmpstr .= ",romfile=$romfile" if $romfile;
1802     }
1803
1804     return $tmpstr;
1805 }
1806
1807 sub print_netdev_full {
1808     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1809
1810     my $i = '';
1811     if ($netid =~ m/^net(\d+)$/) {
1812         $i = int($1);
1813     }
1814
1815     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1816
1817     my $ifname = "tap${vmid}i$i";
1818
1819     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1820     die "interface name '$ifname' is too long (max 15 character)\n"
1821         if length($ifname) >= 16;
1822
1823     my $vhostparam = '';
1824     if (is_native($arch)) {
1825         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1826     }
1827
1828     my $vmname = $conf->{name} || "vm$vmid";
1829
1830     my $netdev = "";
1831     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1832
1833     if ($net->{bridge}) {
1834         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1835             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1836     } else {
1837         $netdev = "type=user,id=$netid,hostname=$vmname";
1838     }
1839
1840     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1841
1842     return $netdev;
1843 }
1844
1845 my $vga_map = {
1846     'cirrus' => 'cirrus-vga',
1847     'std' => 'VGA',
1848     'vmware' => 'vmware-svga',
1849     'virtio' => 'virtio-vga',
1850     'virtio-gl' => 'virtio-vga-gl',
1851 };
1852
1853 sub print_vga_device {
1854     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1855
1856     my $type = $vga_map->{$vga->{type}};
1857     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1858         $type = 'virtio-gpu';
1859     }
1860     my $vgamem_mb = $vga->{memory};
1861
1862     my $max_outputs = '';
1863     if ($qxlnum) {
1864         $type = $id ? 'qxl' : 'qxl-vga';
1865
1866         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1867             # set max outputs so linux can have up to 4 qxl displays with one device
1868             if (min_version($machine_version, 4, 1)) {
1869                 $max_outputs = ",max_outputs=4";
1870             }
1871         }
1872     }
1873
1874     die "no devicetype for $vga->{type}\n" if !$type;
1875
1876     my $memory = "";
1877     if ($vgamem_mb) {
1878         if ($vga->{type} =~ /^virtio/) {
1879             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1880             $memory = ",max_hostmem=$bytes";
1881         } elsif ($qxlnum) {
1882             # from https://www.spice-space.org/multiple-monitors.html
1883             $memory = ",vgamem_mb=$vga->{memory}";
1884             my $ram = $vgamem_mb * 4;
1885             my $vram = $vgamem_mb * 2;
1886             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1887         } else {
1888             $memory = ",vgamem_mb=$vga->{memory}";
1889         }
1890     } elsif ($qxlnum && $id) {
1891         $memory = ",ram_size=67108864,vram_size=33554432";
1892     }
1893
1894     my $edidoff = "";
1895     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1896         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1897     }
1898
1899     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1900     my $vgaid = "vga" . ($id // '');
1901     my $pciaddr;
1902     if ($q35 && $vgaid eq 'vga') {
1903         # the first display uses pcie.0 bus on q35 machines
1904         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1905     } else {
1906         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1907     }
1908
1909     if ($vga->{type} eq 'virtio-gl') {
1910         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1911         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1912             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1913
1914         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1915             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1916     }
1917
1918     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1919 }
1920
1921 sub parse_number_sets {
1922     my ($set) = @_;
1923     my $res = [];
1924     foreach my $part (split(/;/, $set)) {
1925         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1926             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1927             push @$res, [ $1, $2 ];
1928         } else {
1929             die "invalid range: $part\n";
1930         }
1931     }
1932     return $res;
1933 }
1934
1935 sub parse_numa {
1936     my ($data) = @_;
1937
1938     my $res = parse_property_string($numa_fmt, $data);
1939     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1940     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1941     return $res;
1942 }
1943
1944 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1945 sub parse_net {
1946     my ($data, $disable_mac_autogen) = @_;
1947
1948     my $res = eval { parse_property_string($net_fmt, $data) };
1949     if ($@) {
1950         warn $@;
1951         return;
1952     }
1953     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1954         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1955         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1956     }
1957     return $res;
1958 }
1959
1960 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1961 sub parse_ipconfig {
1962     my ($data) = @_;
1963
1964     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1965     if ($@) {
1966         warn $@;
1967         return;
1968     }
1969
1970     if ($res->{gw} && !$res->{ip}) {
1971         warn 'gateway specified without specifying an IP address';
1972         return;
1973     }
1974     if ($res->{gw6} && !$res->{ip6}) {
1975         warn 'IPv6 gateway specified without specifying an IPv6 address';
1976         return;
1977     }
1978     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1979         warn 'gateway specified together with DHCP';
1980         return;
1981     }
1982     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1983         # gw6 + auto/dhcp
1984         warn "IPv6 gateway specified together with $res->{ip6} address";
1985         return;
1986     }
1987
1988     if (!$res->{ip} && !$res->{ip6}) {
1989         return { ip => 'dhcp', ip6 => 'dhcp' };
1990     }
1991
1992     return $res;
1993 }
1994
1995 sub print_net {
1996     my $net = shift;
1997
1998     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1999 }
2000
2001 sub add_random_macs {
2002     my ($settings) = @_;
2003
2004     foreach my $opt (keys %$settings) {
2005         next if $opt !~ m/^net(\d+)$/;
2006         my $net = parse_net($settings->{$opt});
2007         next if !$net;
2008         $settings->{$opt} = print_net($net);
2009     }
2010 }
2011
2012 sub vm_is_volid_owner {
2013     my ($storecfg, $vmid, $volid) = @_;
2014
2015     if ($volid !~  m|^/|) {
2016         my ($path, $owner);
2017         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2018         if ($owner && ($owner == $vmid)) {
2019             return 1;
2020         }
2021     }
2022
2023     return;
2024 }
2025
2026 sub vmconfig_register_unused_drive {
2027     my ($storecfg, $vmid, $conf, $drive) = @_;
2028
2029     if (drive_is_cloudinit($drive)) {
2030         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2031         warn $@ if $@;
2032         delete $conf->{cloudinit};
2033     } elsif (!drive_is_cdrom($drive)) {
2034         my $volid = $drive->{file};
2035         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2036             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2037         }
2038     }
2039 }
2040
2041 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2042 my $smbios1_fmt = {
2043     uuid => {
2044         type => 'string',
2045         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2046         format_description => 'UUID',
2047         description => "Set SMBIOS1 UUID.",
2048         optional => 1,
2049     },
2050     version => {
2051         type => 'string',
2052         pattern => '[A-Za-z0-9+\/]+={0,2}',
2053         format_description => 'Base64 encoded string',
2054         description => "Set SMBIOS1 version.",
2055         optional => 1,
2056     },
2057     serial => {
2058         type => 'string',
2059         pattern => '[A-Za-z0-9+\/]+={0,2}',
2060         format_description => 'Base64 encoded string',
2061         description => "Set SMBIOS1 serial number.",
2062         optional => 1,
2063     },
2064     manufacturer => {
2065         type => 'string',
2066         pattern => '[A-Za-z0-9+\/]+={0,2}',
2067         format_description => 'Base64 encoded string',
2068         description => "Set SMBIOS1 manufacturer.",
2069         optional => 1,
2070     },
2071     product => {
2072         type => 'string',
2073         pattern => '[A-Za-z0-9+\/]+={0,2}',
2074         format_description => 'Base64 encoded string',
2075         description => "Set SMBIOS1 product ID.",
2076         optional => 1,
2077     },
2078     sku => {
2079         type => 'string',
2080         pattern => '[A-Za-z0-9+\/]+={0,2}',
2081         format_description => 'Base64 encoded string',
2082         description => "Set SMBIOS1 SKU string.",
2083         optional => 1,
2084     },
2085     family => {
2086         type => 'string',
2087         pattern => '[A-Za-z0-9+\/]+={0,2}',
2088         format_description => 'Base64 encoded string',
2089         description => "Set SMBIOS1 family string.",
2090         optional => 1,
2091     },
2092     base64 => {
2093         type => 'boolean',
2094         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2095         optional => 1,
2096     },
2097 };
2098
2099 sub parse_smbios1 {
2100     my ($data) = @_;
2101
2102     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2103     warn $@ if $@;
2104     return $res;
2105 }
2106
2107 sub print_smbios1 {
2108     my ($smbios1) = @_;
2109     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2110 }
2111
2112 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2113
2114 sub parse_watchdog {
2115     my ($value) = @_;
2116
2117     return if !$value;
2118
2119     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2120     warn $@ if $@;
2121     return $res;
2122 }
2123
2124 sub parse_guest_agent {
2125     my ($conf) = @_;
2126
2127     return {} if !defined($conf->{agent});
2128
2129     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2130     warn $@ if $@;
2131
2132     # if the agent is disabled ignore the other potentially set properties
2133     return {} if !$res->{enabled};
2134     return $res;
2135 }
2136
2137 sub get_qga_key {
2138     my ($conf, $key) = @_;
2139     return undef if !defined($conf->{agent});
2140
2141     my $agent = parse_guest_agent($conf);
2142     return $agent->{$key};
2143 }
2144
2145 sub parse_vga {
2146     my ($value) = @_;
2147
2148     return {} if !$value;
2149     my $res = eval { parse_property_string($vga_fmt, $value) };
2150     warn $@ if $@;
2151     return $res;
2152 }
2153
2154 sub parse_rng {
2155     my ($value) = @_;
2156
2157     return if !$value;
2158
2159     my $res = eval { parse_property_string($rng_fmt, $value) };
2160     warn $@ if $@;
2161     return $res;
2162 }
2163
2164 sub parse_meta_info {
2165     my ($value) = @_;
2166
2167     return if !$value;
2168
2169     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2170     warn $@ if $@;
2171     return $res;
2172 }
2173
2174 sub new_meta_info_string {
2175     my () = @_; # for now do not allow to override any value
2176
2177     return PVE::JSONSchema::print_property_string(
2178         {
2179             'creation-qemu' => kvm_user_version(),
2180             ctime => "". int(time()),
2181         },
2182         $meta_info_fmt
2183     );
2184 }
2185
2186 sub qemu_created_version_fixups {
2187     my ($conf, $forcemachine, $kvmver) = @_;
2188
2189     my $meta = parse_meta_info($conf->{meta}) // {};
2190     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2191
2192     # check if we need to apply some handling for VMs that always use the latest machine version but
2193     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2194     # would be required (we do not want to pin machine version for non-windows OS type)
2195     if (
2196         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2197         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2198         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2199         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2200     ) {
2201         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2202         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2203             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2204             # and thus with the predictable interface naming of systemd
2205             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2206         }
2207     }
2208     return;
2209 }
2210
2211 # add JSON properties for create and set function
2212 sub json_config_properties {
2213     my ($prop, $with_disk_alloc) = @_;
2214
2215     my $skip_json_config_opts = {
2216         parent => 1,
2217         snaptime => 1,
2218         vmstate => 1,
2219         runningmachine => 1,
2220         runningcpu => 1,
2221         meta => 1,
2222     };
2223
2224     foreach my $opt (keys %$confdesc) {
2225         next if $skip_json_config_opts->{$opt};
2226
2227         if ($with_disk_alloc && is_valid_drivename($opt)) {
2228             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2229         } else {
2230             $prop->{$opt} = $confdesc->{$opt};
2231         }
2232     }
2233
2234     return $prop;
2235 }
2236
2237 # Properties that we can read from an OVF file
2238 sub json_ovf_properties {
2239     my $prop = {};
2240
2241     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2242         $prop->{$device} = {
2243             type => 'string',
2244             format => 'pve-volume-id-or-absolute-path',
2245             description => "Disk image that gets imported to $device",
2246             optional => 1,
2247         };
2248     }
2249
2250     $prop->{cores} = {
2251         type => 'integer',
2252         description => "The number of CPU cores.",
2253         optional => 1,
2254     };
2255     $prop->{memory} = {
2256         type => 'integer',
2257         description => "Amount of RAM for the VM in MB.",
2258         optional => 1,
2259     };
2260     $prop->{name} = {
2261         type => 'string',
2262         description => "Name of the VM.",
2263         optional => 1,
2264     };
2265
2266     return $prop;
2267 }
2268
2269 # return copy of $confdesc_cloudinit to generate documentation
2270 sub cloudinit_config_properties {
2271
2272     return dclone($confdesc_cloudinit);
2273 }
2274
2275 sub cloudinit_pending_properties {
2276     my $p = {
2277         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2278         name => 1,
2279     };
2280     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2281     return $p;
2282 }
2283
2284 sub check_type {
2285     my ($key, $value) = @_;
2286
2287     die "unknown setting '$key'\n" if !$confdesc->{$key};
2288
2289     my $type = $confdesc->{$key}->{type};
2290
2291     if (!defined($value)) {
2292         die "got undefined value\n";
2293     }
2294
2295     if ($value =~ m/[\n\r]/) {
2296         die "property contains a line feed\n";
2297     }
2298
2299     if ($type eq 'boolean') {
2300         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2301         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2302         die "type check ('boolean') failed - got '$value'\n";
2303     } elsif ($type eq 'integer') {
2304         return int($1) if $value =~ m/^(\d+)$/;
2305         die "type check ('integer') failed - got '$value'\n";
2306     } elsif ($type eq 'number') {
2307         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2308         die "type check ('number') failed - got '$value'\n";
2309     } elsif ($type eq 'string') {
2310         if (my $fmt = $confdesc->{$key}->{format}) {
2311             PVE::JSONSchema::check_format($fmt, $value);
2312             return $value;
2313         }
2314         $value =~ s/^\"(.*)\"$/$1/;
2315         return $value;
2316     } else {
2317         die "internal error"
2318     }
2319 }
2320
2321 sub destroy_vm {
2322     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2323
2324     my $conf = PVE::QemuConfig->load_config($vmid);
2325
2326     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2327         PVE::QemuConfig->check_lock($conf);
2328     }
2329
2330     if ($conf->{template}) {
2331         # check if any base image is still used by a linked clone
2332         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2333                 my ($ds, $drive) = @_;
2334                 return if drive_is_cdrom($drive);
2335
2336                 my $volid = $drive->{file};
2337                 return if !$volid || $volid =~ m|^/|;
2338
2339                 die "base volume '$volid' is still in use by linked cloned\n"
2340                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2341
2342         });
2343     }
2344
2345     my $volids = {};
2346     my $remove_owned_drive = sub {
2347         my ($ds, $drive) = @_;
2348         return if drive_is_cdrom($drive, 1);
2349
2350         my $volid = $drive->{file};
2351         return if !$volid || $volid =~ m|^/|;
2352         return if $volids->{$volid};
2353
2354         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2355         return if !$path || !$owner || ($owner != $vmid);
2356
2357         $volids->{$volid} = 1;
2358         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2359         warn "Could not remove disk '$volid', check manually: $@" if $@;
2360     };
2361
2362     # only remove disks owned by this VM (referenced in the config)
2363     my $include_opts = {
2364         include_unused => 1,
2365         extra_keys => ['vmstate'],
2366     };
2367     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2368
2369     for my $snap (values %{$conf->{snapshots}}) {
2370         next if !defined($snap->{vmstate});
2371         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2372         next if !defined($drive);
2373         $remove_owned_drive->('vmstate', $drive);
2374     }
2375
2376     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2377
2378     if ($purge_unreferenced) { # also remove unreferenced disk
2379         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2380         PVE::Storage::foreach_volid($vmdisks, sub {
2381             my ($volid, $sid, $volname, $d) = @_;
2382             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2383             warn $@ if $@;
2384         });
2385     }
2386
2387     if (defined $replacement_conf) {
2388         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2389     } else {
2390         PVE::QemuConfig->destroy_config($vmid);
2391     }
2392 }
2393
2394 sub parse_vm_config {
2395     my ($filename, $raw, $strict) = @_;
2396
2397     return if !defined($raw);
2398
2399     my $res = {
2400         digest => Digest::SHA::sha1_hex($raw),
2401         snapshots => {},
2402         pending => {},
2403         cloudinit => {},
2404     };
2405
2406     my $handle_error = sub {
2407         my ($msg) = @_;
2408
2409         if ($strict) {
2410             die $msg;
2411         } else {
2412             warn $msg;
2413         }
2414     };
2415
2416     $filename =~ m|/qemu-server/(\d+)\.conf$|
2417         || die "got strange filename '$filename'";
2418
2419     my $vmid = $1;
2420
2421     my $conf = $res;
2422     my $descr;
2423     my $finish_description = sub {
2424         if (defined($descr)) {
2425             $descr =~ s/\s+$//;
2426             $conf->{description} = $descr;
2427         }
2428         $descr = undef;
2429     };
2430     my $section = '';
2431
2432     my @lines = split(/\n/, $raw);
2433     foreach my $line (@lines) {
2434         next if $line =~ m/^\s*$/;
2435
2436         if ($line =~ m/^\[PENDING\]\s*$/i) {
2437             $section = 'pending';
2438             $finish_description->();
2439             $conf = $res->{$section} = {};
2440             next;
2441         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2442             $section = 'cloudinit';
2443             $finish_description->();
2444             $conf = $res->{$section} = {};
2445             next;
2446
2447         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2448             $section = $1;
2449             $finish_description->();
2450             $conf = $res->{snapshots}->{$section} = {};
2451             next;
2452         }
2453
2454         if ($line =~ m/^\#(.*)$/) {
2455             $descr = '' if !defined($descr);
2456             $descr .= PVE::Tools::decode_text($1) . "\n";
2457             next;
2458         }
2459
2460         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2461             $descr = '' if !defined($descr);
2462             $descr .= PVE::Tools::decode_text($2);
2463         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2464             $conf->{snapstate} = $1;
2465         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2466             my $key = $1;
2467             my $value = $2;
2468             $conf->{$key} = $value;
2469         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2470             my $value = $1;
2471             if ($section eq 'pending') {
2472                 $conf->{delete} = $value; # we parse this later
2473             } else {
2474                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2475             }
2476         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2477             my $key = $1;
2478             my $value = $2;
2479             if ($section eq 'cloudinit') {
2480                 # ignore validation only used for informative purpose
2481                 $conf->{$key} = $value;
2482                 next;
2483             }
2484             eval { $value = check_type($key, $value); };
2485             if ($@) {
2486                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2487             } else {
2488                 $key = 'ide2' if $key eq 'cdrom';
2489                 my $fmt = $confdesc->{$key}->{format};
2490                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2491                     my $v = parse_drive($key, $value);
2492                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2493                         $v->{file} = $volid;
2494                         $value = print_drive($v);
2495                     } else {
2496                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2497                         next;
2498                     }
2499                 }
2500
2501                 $conf->{$key} = $value;
2502             }
2503         } else {
2504             $handle_error->("vm $vmid - unable to parse config: $line\n");
2505         }
2506     }
2507
2508     $finish_description->();
2509     delete $res->{snapstate}; # just to be sure
2510
2511     return $res;
2512 }
2513
2514 sub write_vm_config {
2515     my ($filename, $conf) = @_;
2516
2517     delete $conf->{snapstate}; # just to be sure
2518
2519     if ($conf->{cdrom}) {
2520         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2521         $conf->{ide2} = $conf->{cdrom};
2522         delete $conf->{cdrom};
2523     }
2524
2525     # we do not use 'smp' any longer
2526     if ($conf->{sockets}) {
2527         delete $conf->{smp};
2528     } elsif ($conf->{smp}) {
2529         $conf->{sockets} = $conf->{smp};
2530         delete $conf->{cores};
2531         delete $conf->{smp};
2532     }
2533
2534     my $used_volids = {};
2535
2536     my $cleanup_config = sub {
2537         my ($cref, $pending, $snapname) = @_;
2538
2539         foreach my $key (keys %$cref) {
2540             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2541                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2542             my $value = $cref->{$key};
2543             if ($key eq 'delete') {
2544                 die "propertry 'delete' is only allowed in [PENDING]\n"
2545                     if !$pending;
2546                 # fixme: check syntax?
2547                 next;
2548             }
2549             eval { $value = check_type($key, $value); };
2550             die "unable to parse value of '$key' - $@" if $@;
2551
2552             $cref->{$key} = $value;
2553
2554             if (!$snapname && is_valid_drivename($key)) {
2555                 my $drive = parse_drive($key, $value);
2556                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2557             }
2558         }
2559     };
2560
2561     &$cleanup_config($conf);
2562
2563     &$cleanup_config($conf->{pending}, 1);
2564
2565     foreach my $snapname (keys %{$conf->{snapshots}}) {
2566         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2567         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2568     }
2569
2570     # remove 'unusedX' settings if we re-add a volume
2571     foreach my $key (keys %$conf) {
2572         my $value = $conf->{$key};
2573         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2574             delete $conf->{$key};
2575         }
2576     }
2577
2578     my $generate_raw_config = sub {
2579         my ($conf, $pending) = @_;
2580
2581         my $raw = '';
2582
2583         # add description as comment to top of file
2584         if (defined(my $descr = $conf->{description})) {
2585             if ($descr) {
2586                 foreach my $cl (split(/\n/, $descr)) {
2587                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2588                 }
2589             } else {
2590                 $raw .= "#\n" if $pending;
2591             }
2592         }
2593
2594         foreach my $key (sort keys %$conf) {
2595             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2596             $raw .= "$key: $conf->{$key}\n";
2597         }
2598         return $raw;
2599     };
2600
2601     my $raw = &$generate_raw_config($conf);
2602
2603     if (scalar(keys %{$conf->{pending}})){
2604         $raw .= "\n[PENDING]\n";
2605         $raw .= &$generate_raw_config($conf->{pending}, 1);
2606     }
2607
2608     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2609         $raw .= "\n[special:cloudinit]\n";
2610         $raw .= &$generate_raw_config($conf->{cloudinit});
2611     }
2612
2613     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2614         $raw .= "\n[$snapname]\n";
2615         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2616     }
2617
2618     return $raw;
2619 }
2620
2621 sub load_defaults {
2622
2623     my $res = {};
2624
2625     # we use static defaults from our JSON schema configuration
2626     foreach my $key (keys %$confdesc) {
2627         if (defined(my $default = $confdesc->{$key}->{default})) {
2628             $res->{$key} = $default;
2629         }
2630     }
2631
2632     return $res;
2633 }
2634
2635 sub config_list {
2636     my $vmlist = PVE::Cluster::get_vmlist();
2637     my $res = {};
2638     return $res if !$vmlist || !$vmlist->{ids};
2639     my $ids = $vmlist->{ids};
2640     my $nodename = nodename();
2641
2642     foreach my $vmid (keys %$ids) {
2643         my $d = $ids->{$vmid};
2644         next if !$d->{node} || $d->{node} ne $nodename;
2645         next if !$d->{type} || $d->{type} ne 'qemu';
2646         $res->{$vmid}->{exists} = 1;
2647     }
2648     return $res;
2649 }
2650
2651 # test if VM uses local resources (to prevent migration)
2652 sub check_local_resources {
2653     my ($conf, $noerr) = @_;
2654
2655     my @loc_res = ();
2656     my $mapped_res = [];
2657
2658     my $nodelist = PVE::Cluster::get_nodelist();
2659     my $pci_map = PVE::Mapping::PCI::config();
2660     my $usb_map = PVE::Mapping::USB::config();
2661
2662     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2663
2664     my $add_missing_mapping = sub {
2665         my ($type, $key, $id) = @_;
2666         for my $node (@$nodelist) {
2667             my $entry;
2668             if ($type eq 'pci') {
2669                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2670             } elsif ($type eq 'usb') {
2671                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2672             }
2673             if (!scalar($entry->@*)) {
2674                 push @{$missing_mappings_by_node->{$node}}, $key;
2675             }
2676         }
2677     };
2678
2679     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2680     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2681
2682     push @loc_res, "ivshmem" if $conf->{ivshmem};
2683
2684     foreach my $k (keys %$conf) {
2685         if ($k =~ m/^usb/) {
2686             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2687             next if $entry->{host} =~ m/^spice$/i;
2688             if ($entry->{mapping}) {
2689                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2690                 push @$mapped_res, $k;
2691             }
2692         }
2693         if ($k =~ m/^hostpci/) {
2694             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2695             if ($entry->{mapping}) {
2696                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2697                 push @$mapped_res, $k;
2698             }
2699         }
2700         # sockets are safe: they will recreated be on the target side post-migrate
2701         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2702         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2703     }
2704
2705     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2706
2707     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2708 }
2709
2710 # check if used storages are available on all nodes (use by migrate)
2711 sub check_storage_availability {
2712     my ($storecfg, $conf, $node) = @_;
2713
2714     PVE::QemuConfig->foreach_volume($conf, sub {
2715         my ($ds, $drive) = @_;
2716
2717         my $volid = $drive->{file};
2718         return if !$volid;
2719
2720         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2721         return if !$sid;
2722
2723         # check if storage is available on both nodes
2724         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2725         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2726
2727         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2728
2729         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2730             if !$scfg->{content}->{$vtype};
2731    });
2732 }
2733
2734 # list nodes where all VM images are available (used by has_feature API)
2735 sub shared_nodes {
2736     my ($conf, $storecfg) = @_;
2737
2738     my $nodelist = PVE::Cluster::get_nodelist();
2739     my $nodehash = { map { $_ => 1 } @$nodelist };
2740     my $nodename = nodename();
2741
2742     PVE::QemuConfig->foreach_volume($conf, sub {
2743         my ($ds, $drive) = @_;
2744
2745         my $volid = $drive->{file};
2746         return if !$volid;
2747
2748         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2749         if ($storeid) {
2750             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2751             if ($scfg->{disable}) {
2752                 $nodehash = {};
2753             } elsif (my $avail = $scfg->{nodes}) {
2754                 foreach my $node (keys %$nodehash) {
2755                     delete $nodehash->{$node} if !$avail->{$node};
2756                 }
2757             } elsif (!$scfg->{shared}) {
2758                 foreach my $node (keys %$nodehash) {
2759                     delete $nodehash->{$node} if $node ne $nodename
2760                 }
2761             }
2762         }
2763     });
2764
2765     return $nodehash
2766 }
2767
2768 sub check_local_storage_availability {
2769     my ($conf, $storecfg) = @_;
2770
2771     my $nodelist = PVE::Cluster::get_nodelist();
2772     my $nodehash = { map { $_ => {} } @$nodelist };
2773
2774     PVE::QemuConfig->foreach_volume($conf, sub {
2775         my ($ds, $drive) = @_;
2776
2777         my $volid = $drive->{file};
2778         return if !$volid;
2779
2780         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2781         if ($storeid) {
2782             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2783
2784             if ($scfg->{disable}) {
2785                 foreach my $node (keys %$nodehash) {
2786                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2787                 }
2788             } elsif (my $avail = $scfg->{nodes}) {
2789                 foreach my $node (keys %$nodehash) {
2790                     if (!$avail->{$node}) {
2791                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2792                     }
2793                 }
2794             }
2795         }
2796     });
2797
2798     foreach my $node (values %$nodehash) {
2799         if (my $unavail = $node->{unavailable_storages}) {
2800             $node->{unavailable_storages} = [ sort keys %$unavail ];
2801         }
2802     }
2803
2804     return $nodehash
2805 }
2806
2807 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2808 sub check_running {
2809     my ($vmid, $nocheck, $node) = @_;
2810
2811     # $nocheck is set when called during a migration, in which case the config
2812     # file might still or already reside on the *other* node
2813     # - because rename has already happened, and current node is source
2814     # - because rename hasn't happened yet, and current node is target
2815     # - because rename has happened, current node is target, but hasn't yet
2816     # processed it yet
2817     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2818     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2819 }
2820
2821 sub vzlist {
2822
2823     my $vzlist = config_list();
2824
2825     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2826
2827     while (defined(my $de = $fd->read)) {
2828         next if $de !~ m/^(\d+)\.pid$/;
2829         my $vmid = $1;
2830         next if !defined($vzlist->{$vmid});
2831         if (my $pid = check_running($vmid)) {
2832             $vzlist->{$vmid}->{pid} = $pid;
2833         }
2834     }
2835
2836     return $vzlist;
2837 }
2838
2839 our $vmstatus_return_properties = {
2840     vmid => get_standard_option('pve-vmid'),
2841     status => {
2842         description => "QEMU process status.",
2843         type => 'string',
2844         enum => ['stopped', 'running'],
2845     },
2846     maxmem => {
2847         description => "Maximum memory in bytes.",
2848         type => 'integer',
2849         optional => 1,
2850         renderer => 'bytes',
2851     },
2852     maxdisk => {
2853         description => "Root disk size in bytes.",
2854         type => 'integer',
2855         optional => 1,
2856         renderer => 'bytes',
2857     },
2858     name => {
2859         description => "VM name.",
2860         type => 'string',
2861         optional => 1,
2862     },
2863     qmpstatus => {
2864         description => "VM run state from the 'query-status' QMP monitor command.",
2865         type => 'string',
2866         optional => 1,
2867     },
2868     pid => {
2869         description => "PID of running qemu process.",
2870         type => 'integer',
2871         optional => 1,
2872     },
2873     uptime => {
2874         description => "Uptime.",
2875         type => 'integer',
2876         optional => 1,
2877         renderer => 'duration',
2878     },
2879     cpus => {
2880         description => "Maximum usable CPUs.",
2881         type => 'number',
2882         optional => 1,
2883     },
2884     lock => {
2885         description => "The current config lock, if any.",
2886         type => 'string',
2887         optional => 1,
2888     },
2889     tags => {
2890         description  => "The current configured tags, if any",
2891         type => 'string',
2892         optional => 1,
2893     },
2894     'running-machine' => {
2895         description => "The currently running machine type (if running).",
2896         type => 'string',
2897         optional => 1,
2898     },
2899     'running-qemu' => {
2900         description => "The currently running QEMU version (if running).",
2901         type => 'string',
2902         optional => 1,
2903     },
2904 };
2905
2906 my $last_proc_pid_stat;
2907
2908 # get VM status information
2909 # This must be fast and should not block ($full == false)
2910 # We only query KVM using QMP if $full == true (this can be slow)
2911 sub vmstatus {
2912     my ($opt_vmid, $full) = @_;
2913
2914     my $res = {};
2915
2916     my $storecfg = PVE::Storage::config();
2917
2918     my $list = vzlist();
2919     my $defaults = load_defaults();
2920
2921     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2922
2923     my $cpucount = $cpuinfo->{cpus} || 1;
2924
2925     foreach my $vmid (keys %$list) {
2926         next if $opt_vmid && ($vmid ne $opt_vmid);
2927
2928         my $conf = PVE::QemuConfig->load_config($vmid);
2929
2930         my $d = { vmid => int($vmid) };
2931         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2932
2933         # fixme: better status?
2934         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2935
2936         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2937         if (defined($size)) {
2938             $d->{disk} = 0; # no info available
2939             $d->{maxdisk} = $size;
2940         } else {
2941             $d->{disk} = 0;
2942             $d->{maxdisk} = 0;
2943         }
2944
2945         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2946             * ($conf->{cores} || $defaults->{cores});
2947         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2948         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2949
2950         $d->{name} = $conf->{name} || "VM $vmid";
2951         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2952             : $defaults->{memory}*(1024*1024);
2953
2954         if ($conf->{balloon}) {
2955             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2956             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2957                 : $defaults->{shares};
2958         }
2959
2960         $d->{uptime} = 0;
2961         $d->{cpu} = 0;
2962         $d->{mem} = 0;
2963
2964         $d->{netout} = 0;
2965         $d->{netin} = 0;
2966
2967         $d->{diskread} = 0;
2968         $d->{diskwrite} = 0;
2969
2970         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2971
2972         $d->{serial} = 1 if conf_has_serial($conf);
2973         $d->{lock} = $conf->{lock} if $conf->{lock};
2974         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2975
2976         $res->{$vmid} = $d;
2977     }
2978
2979     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2980     foreach my $dev (keys %$netdev) {
2981         next if $dev !~ m/^tap([1-9]\d*)i/;
2982         my $vmid = $1;
2983         my $d = $res->{$vmid};
2984         next if !$d;
2985
2986         $d->{netout} += $netdev->{$dev}->{receive};
2987         $d->{netin} += $netdev->{$dev}->{transmit};
2988
2989         if ($full) {
2990             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2991             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2992         }
2993
2994     }
2995
2996     my $ctime = gettimeofday;
2997
2998     foreach my $vmid (keys %$list) {
2999
3000         my $d = $res->{$vmid};
3001         my $pid = $d->{pid};
3002         next if !$pid;
3003
3004         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3005         next if !$pstat; # not running
3006
3007         my $used = $pstat->{utime} + $pstat->{stime};
3008
3009         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3010
3011         if ($pstat->{vsize}) {
3012             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3013         }
3014
3015         my $old = $last_proc_pid_stat->{$pid};
3016         if (!$old) {
3017             $last_proc_pid_stat->{$pid} = {
3018                 time => $ctime,
3019                 used => $used,
3020                 cpu => 0,
3021             };
3022             next;
3023         }
3024
3025         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3026
3027         if ($dtime > 1000) {
3028             my $dutime = $used -  $old->{used};
3029
3030             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3031             $last_proc_pid_stat->{$pid} = {
3032                 time => $ctime,
3033                 used => $used,
3034                 cpu => $d->{cpu},
3035             };
3036         } else {
3037             $d->{cpu} = $old->{cpu};
3038         }
3039     }
3040
3041     return $res if !$full;
3042
3043     my $qmpclient = PVE::QMPClient->new();
3044
3045     my $ballooncb = sub {
3046         my ($vmid, $resp) = @_;
3047
3048         my $info = $resp->{'return'};
3049         return if !$info->{max_mem};
3050
3051         my $d = $res->{$vmid};
3052
3053         # use memory assigned to VM
3054         $d->{maxmem} = $info->{max_mem};
3055         $d->{balloon} = $info->{actual};
3056
3057         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3058             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3059             $d->{freemem} = $info->{free_mem};
3060         }
3061
3062         $d->{ballooninfo} = $info;
3063     };
3064
3065     my $blockstatscb = sub {
3066         my ($vmid, $resp) = @_;
3067         my $data = $resp->{'return'} || [];
3068         my $totalrdbytes = 0;
3069         my $totalwrbytes = 0;
3070
3071         for my $blockstat (@$data) {
3072             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3073             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3074
3075             $blockstat->{device} =~ s/drive-//;
3076             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3077         }
3078         $res->{$vmid}->{diskread} = $totalrdbytes;
3079         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3080     };
3081
3082     my $machinecb = sub {
3083         my ($vmid, $resp) = @_;
3084         my $data = $resp->{'return'} || [];
3085
3086         $res->{$vmid}->{'running-machine'} =
3087             PVE::QemuServer::Machine::current_from_query_machines($data);
3088     };
3089
3090     my $versioncb = sub {
3091         my ($vmid, $resp) = @_;
3092         my $data = $resp->{'return'} // {};
3093         my $version = 'unknown';
3094
3095         if (my $v = $data->{qemu}) {
3096             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3097         }
3098
3099         $res->{$vmid}->{'running-qemu'} = $version;
3100     };
3101
3102     my $statuscb = sub {
3103         my ($vmid, $resp) = @_;
3104
3105         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3106         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3107         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3108         # this fails if ballon driver is not loaded, so this must be
3109         # the last commnand (following command are aborted if this fails).
3110         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3111
3112         my $status = 'unknown';
3113         if (!defined($status = $resp->{'return'}->{status})) {
3114             warn "unable to get VM status\n";
3115             return;
3116         }
3117
3118         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3119     };
3120
3121     foreach my $vmid (keys %$list) {
3122         next if $opt_vmid && ($vmid ne $opt_vmid);
3123         next if !$res->{$vmid}->{pid}; # not running
3124         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3125     }
3126
3127     $qmpclient->queue_execute(undef, 2);
3128
3129     foreach my $vmid (keys %$list) {
3130         next if $opt_vmid && ($vmid ne $opt_vmid);
3131         next if !$res->{$vmid}->{pid}; #not running
3132
3133         # we can't use the $qmpclient since it might have already aborted on
3134         # 'query-balloon', but this might also fail for older versions...
3135         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3136         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3137     }
3138
3139     foreach my $vmid (keys %$list) {
3140         next if $opt_vmid && ($vmid ne $opt_vmid);
3141         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3142     }
3143
3144     return $res;
3145 }
3146
3147 sub conf_has_serial {
3148     my ($conf) = @_;
3149
3150     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3151         if ($conf->{"serial$i"}) {
3152             return 1;
3153         }
3154     }
3155
3156     return 0;
3157 }
3158
3159 sub conf_has_audio {
3160     my ($conf, $id) = @_;
3161
3162     $id //= 0;
3163     my $audio = $conf->{"audio$id"};
3164     return if !defined($audio);
3165
3166     my $audioproperties = parse_property_string($audio_fmt, $audio);
3167     my $audiodriver = $audioproperties->{driver} // 'spice';
3168
3169     return {
3170         dev => $audioproperties->{device},
3171         dev_id => "audiodev$id",
3172         backend => $audiodriver,
3173         backend_id => "$audiodriver-backend${id}",
3174     };
3175 }
3176
3177 sub audio_devs {
3178     my ($audio, $audiopciaddr, $machine_version) = @_;
3179
3180     my $devs = [];
3181
3182     my $id = $audio->{dev_id};
3183     my $audiodev = "";
3184     if (min_version($machine_version, 4, 2)) {
3185         $audiodev = ",audiodev=$audio->{backend_id}";
3186     }
3187
3188     if ($audio->{dev} eq 'AC97') {
3189         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3190     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3191         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3192         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3193         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3194     } else {
3195         die "unkown audio device '$audio->{dev}', implement me!";
3196     }
3197
3198     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3199
3200     return $devs;
3201 }
3202
3203 sub get_tpm_paths {
3204     my ($vmid) = @_;
3205     return {
3206         socket => "/var/run/qemu-server/$vmid.swtpm",
3207         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3208     };
3209 }
3210
3211 sub add_tpm_device {
3212     my ($vmid, $devices, $conf) = @_;
3213
3214     return if !$conf->{tpmstate0};
3215
3216     my $paths = get_tpm_paths($vmid);
3217
3218     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3219     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3220     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3221 }
3222
3223 sub start_swtpm {
3224     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3225
3226     return if !$tpmdrive;
3227
3228     my $state;
3229     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3230     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3231     if ($storeid) {
3232         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3233     } else {
3234         $state = $tpm->{file};
3235     }
3236
3237     my $paths = get_tpm_paths($vmid);
3238
3239     # during migration, we will get state from remote
3240     #
3241     if (!$migration) {
3242         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3243         my $setup_cmd = [
3244             "swtpm_setup",
3245             "--tpmstate",
3246             "file://$state",
3247             "--createek",
3248             "--create-ek-cert",
3249             "--create-platform-cert",
3250             "--lock-nvram",
3251             "--config",
3252             "/etc/swtpm_setup.conf", # do not use XDG configs
3253             "--runas",
3254             "0", # force creation as root, error if not possible
3255             "--not-overwrite", # ignore existing state, do not modify
3256         ];
3257
3258         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3259         # TPM 2.0 supports ECC crypto, use if possible
3260         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3261
3262         run_command($setup_cmd, outfunc => sub {
3263             print "swtpm_setup: $1\n";
3264         });
3265     }
3266
3267     # Used to distinguish different invocations in the log.
3268     my $log_prefix = "[id=" . int(time()) . "] ";
3269
3270     my $emulator_cmd = [
3271         "swtpm",
3272         "socket",
3273         "--tpmstate",
3274         "backend-uri=file://$state,mode=0600",
3275         "--ctrl",
3276         "type=unixio,path=$paths->{socket},mode=0600",
3277         "--pid",
3278         "file=$paths->{pid}",
3279         "--terminate", # terminate on QEMU disconnect
3280         "--daemon",
3281         "--log",
3282         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3283     ];
3284     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3285     run_command($emulator_cmd, outfunc => sub { print $1; });
3286
3287     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3288     while (! -e $paths->{pid}) {
3289         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3290         usleep(50_000);
3291     }
3292
3293     # return untainted PID of swtpm daemon so it can be killed on error
3294     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3295     return $1;
3296 }
3297
3298 sub vga_conf_has_spice {
3299     my ($vga) = @_;
3300
3301     my $vgaconf = parse_vga($vga);
3302     my $vgatype = $vgaconf->{type};
3303     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3304
3305     return $1 || 1;
3306 }
3307
3308 sub is_native($) {
3309     my ($arch) = @_;
3310     return get_host_arch() eq $arch;
3311 }
3312
3313 sub get_vm_arch {
3314     my ($conf) = @_;
3315     return $conf->{arch} // get_host_arch();
3316 }
3317
3318 my $default_machines = {
3319     x86_64 => 'pc',
3320     aarch64 => 'virt',
3321 };
3322
3323 sub get_installed_machine_version {
3324     my ($kvmversion) = @_;
3325     $kvmversion = kvm_user_version() if !defined($kvmversion);
3326     $kvmversion =~ m/^(\d+\.\d+)/;
3327     return $1;
3328 }
3329
3330 sub windows_get_pinned_machine_version {
3331     my ($machine, $base_version, $kvmversion) = @_;
3332
3333     my $pin_version = $base_version;
3334     if (!defined($base_version) ||
3335         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3336     ) {
3337         $pin_version = get_installed_machine_version($kvmversion);
3338     }
3339     if (!$machine || $machine eq 'pc') {
3340         $machine = "pc-i440fx-$pin_version";
3341     } elsif ($machine eq 'q35') {
3342         $machine = "pc-q35-$pin_version";
3343     } elsif ($machine eq 'virt') {
3344         $machine = "virt-$pin_version";
3345     } else {
3346         warn "unknown machine type '$machine', not touching that!\n";
3347     }
3348
3349     return $machine;
3350 }
3351
3352 sub get_vm_machine {
3353     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3354
3355     my $machine = $forcemachine || $conf->{machine};
3356
3357     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3358         $kvmversion //= kvm_user_version();
3359         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3360         # layout which confuses windows quite a bit and may result in various regressions..
3361         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3362         if (windows_version($conf->{ostype})) {
3363             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3364         }
3365         $arch //= 'x86_64';
3366         $machine ||= $default_machines->{$arch};
3367         if ($add_pve_version) {
3368             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3369             $machine .= "+pve$pvever";
3370         }
3371     }
3372
3373     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3374         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3375         $machine = $1 if $is_pxe;
3376
3377         # for version-pinned machines that do not include a pve-version (e.g.
3378         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3379         $machine .= '+pve0';
3380
3381         $machine .= '.pxe' if $is_pxe;
3382     }
3383
3384     return $machine;
3385 }
3386
3387 sub get_ovmf_files($$$) {
3388     my ($arch, $efidisk, $smm) = @_;
3389
3390     my $types = $OVMF->{$arch}
3391         or die "no OVMF images known for architecture '$arch'\n";
3392
3393     my $type = 'default';
3394     if ($arch ne "aarch64" && defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3395         $type = $smm ? "4m" : "4m-no-smm";
3396         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3397     }
3398
3399     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3400     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3401     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3402
3403     return ($ovmf_code, $ovmf_vars);
3404 }
3405
3406 my $Arch2Qemu = {
3407     aarch64 => '/usr/bin/qemu-system-aarch64',
3408     x86_64 => '/usr/bin/qemu-system-x86_64',
3409 };
3410 sub get_command_for_arch($) {
3411     my ($arch) = @_;
3412     return '/usr/bin/kvm' if is_native($arch);
3413
3414     my $cmd = $Arch2Qemu->{$arch}
3415         or die "don't know how to emulate architecture '$arch'\n";
3416     return $cmd;
3417 }
3418
3419 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3420 # to use in a QEMU command line (-cpu element), first array_intersect the result
3421 # of query_supported_ with query_understood_. This is necessary because:
3422 #
3423 # a) query_understood_ returns flags the host cannot use and
3424 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3425 #    flags, but CPU settings - with most of them being flags. Those settings
3426 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3427 #
3428 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3429 # expensive. If you need the value returned from this, you can get it much
3430 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3431 # $accel being 'kvm' or 'tcg'.
3432 #
3433 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3434 # changes, automatically populating pmxcfs.
3435 #
3436 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3437 # since kvm and tcg machines support different flags
3438 #
3439 sub query_supported_cpu_flags {
3440     my ($arch) = @_;
3441
3442     $arch //= get_host_arch();
3443     my $default_machine = $default_machines->{$arch};
3444
3445     my $flags = {};
3446
3447     # FIXME: Once this is merged, the code below should work for ARM as well:
3448     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3449     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3450         $arch eq "aarch64";
3451
3452     my $kvm_supported = defined(kvm_version());
3453     my $qemu_cmd = get_command_for_arch($arch);
3454     my $fakevmid = -1;
3455     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3456
3457     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3458     my $query_supported_run_qemu = sub {
3459         my ($kvm) = @_;
3460
3461         my $flags = {};
3462         my $cmd = [
3463             $qemu_cmd,
3464             '-machine', $default_machine,
3465             '-display', 'none',
3466             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3467             '-mon', 'chardev=qmp,mode=control',
3468             '-pidfile', $pidfile,
3469             '-S', '-daemonize'
3470         ];
3471
3472         if (!$kvm) {
3473             push @$cmd, '-accel', 'tcg';
3474         }
3475
3476         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3477         die "QEMU flag querying VM exited with code " . $rc if $rc;
3478
3479         eval {
3480             my $cmd_result = mon_cmd(
3481                 $fakevmid,
3482                 'query-cpu-model-expansion',
3483                 type => 'full',
3484                 model => { name => 'host' }
3485             );
3486
3487             my $props = $cmd_result->{model}->{props};
3488             foreach my $prop (keys %$props) {
3489                 next if $props->{$prop} ne '1';
3490                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3491                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3492                 # We only keep those with underscores, to match /proc/cpuinfo
3493                 $prop =~ s/\.|-/_/g;
3494                 $flags->{$prop} = 1;
3495             }
3496         };
3497         my $err = $@;
3498
3499         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3500         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3501
3502         die $err if $err;
3503
3504         return [ sort keys %$flags ];
3505     };
3506
3507     # We need to query QEMU twice, since KVM and TCG have different supported flags
3508     PVE::QemuConfig->lock_config($fakevmid, sub {
3509         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3510         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3511
3512         if ($kvm_supported) {
3513             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3514             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3515         }
3516     });
3517
3518     return $flags;
3519 }
3520
3521 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3522 my $understood_cpu_flag_dir = "/usr/share/kvm";
3523 sub query_understood_cpu_flags {
3524     my $arch = get_host_arch();
3525     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3526
3527     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3528         if ! -e $filepath;
3529
3530     my $raw = file_get_contents($filepath);
3531     $raw =~ s/^\s+|\s+$//g;
3532     my @flags = split(/\s+/, $raw);
3533
3534     return \@flags;
3535 }
3536
3537 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3538 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3539 my sub should_disable_smm {
3540     my ($conf, $vga, $machine) = @_;
3541
3542     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3543
3544     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3545         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3546 }
3547
3548 my sub print_ovmf_drive_commandlines {
3549     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3550
3551     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3552
3553     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3554
3555     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3556     if ($d) {
3557         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3558         my ($path, $format) = $d->@{'file', 'format'};
3559         if ($storeid) {
3560             $path = PVE::Storage::path($storecfg, $d->{file});
3561             if (!defined($format)) {
3562                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3563                 $format = qemu_img_format($scfg, $volname);
3564             }
3565         } elsif (!defined($format)) {
3566             die "efidisk format must be specified\n";
3567         }
3568         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3569         if ($path =~ m/^rbd:/) {
3570             $var_drive_str .= ',cache=writeback';
3571             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3572         }
3573         $var_drive_str .= ",format=$format,file=$path";
3574
3575         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3576         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3577     } else {
3578         log_warn("no efidisk configured! Using temporary efivars disk.");
3579         my $path = "/tmp/$vmid-ovmf.fd";
3580         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3581         $var_drive_str .= ",format=raw,file=$path";
3582         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3583     }
3584
3585     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3586 }
3587
3588 sub config_to_command {
3589     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3590         $pbs_backing) = @_;
3591
3592     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3593     my $devices = [];
3594     my $bridges = {};
3595     my $ostype = $conf->{ostype};
3596     my $winversion = windows_version($ostype);
3597     my $kvm = $conf->{kvm};
3598     my $nodename = nodename();
3599
3600     my $arch = get_vm_arch($conf);
3601     my $kvm_binary = get_command_for_arch($arch);
3602     my $kvmver = kvm_user_version($kvm_binary);
3603
3604     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3605         $kvmver //= "undefined";
3606         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3607     }
3608
3609     my $add_pve_version = min_version($kvmver, 4, 1);
3610
3611     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3612     my $machine_version = extract_version($machine_type, $kvmver);
3613     $kvm //= 1 if is_native($arch);
3614
3615     $machine_version =~ m/(\d+)\.(\d+)/;
3616     my ($machine_major, $machine_minor) = ($1, $2);
3617
3618     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3619         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3620     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3621         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3622             ." please upgrade node '$nodename'\n"
3623     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3624         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3625         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3626             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3627             ." node '$nodename'\n";
3628     }
3629
3630     # if a specific +pve version is required for a feature, use $version_guard
3631     # instead of min_version to allow machines to be run with the minimum
3632     # required version
3633     my $required_pve_version = 0;
3634     my $version_guard = sub {
3635         my ($major, $minor, $pve) = @_;
3636         return 0 if !min_version($machine_version, $major, $minor, $pve);
3637         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3638         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3639         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3640         return 1;
3641     };
3642
3643     if ($kvm && !defined kvm_version()) {
3644         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3645             ." or enable in BIOS.\n";
3646     }
3647
3648     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3649     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3650     my $use_old_bios_files = undef;
3651     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3652
3653     my $cmd = [];
3654     if ($conf->{affinity}) {
3655         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3656     }
3657
3658     push @$cmd, $kvm_binary;
3659
3660     push @$cmd, '-id', $vmid;
3661
3662     my $vmname = $conf->{name} || "vm$vmid";
3663
3664     push @$cmd, '-name', "$vmname,debug-threads=on";
3665
3666     push @$cmd, '-no-shutdown';
3667
3668     my $use_virtio = 0;
3669
3670     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3671     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3672     push @$cmd, '-mon', "chardev=qmp,mode=control";
3673
3674     if (min_version($machine_version, 2, 12)) {
3675         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3676         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3677     }
3678
3679     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3680
3681     push @$cmd, '-daemonize';
3682
3683     if ($conf->{smbios1}) {
3684         my $smbios_conf = parse_smbios1($conf->{smbios1});
3685         if ($smbios_conf->{base64}) {
3686             # Do not pass base64 flag to qemu
3687             delete $smbios_conf->{base64};
3688             my $smbios_string = "";
3689             foreach my $key (keys %$smbios_conf) {
3690                 my $value;
3691                 if ($key eq "uuid") {
3692                     $value = $smbios_conf->{uuid}
3693                 } else {
3694                     $value = decode_base64($smbios_conf->{$key});
3695                 }
3696                 # qemu accepts any binary data, only commas need escaping by double comma
3697                 $value =~ s/,/,,/g;
3698                 $smbios_string .= "," . $key . "=" . $value if $value;
3699             }
3700             push @$cmd, '-smbios', "type=1" . $smbios_string;
3701         } else {
3702             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3703         }
3704     }
3705
3706     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3707         my ($code_drive_str, $var_drive_str) =
3708             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3709         push $cmd->@*, '-drive', $code_drive_str;
3710         push $cmd->@*, '-drive', $var_drive_str;
3711     }
3712
3713     if ($q35) { # tell QEMU to load q35 config early
3714         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3715         if (min_version($machine_version, 4, 0)) {
3716             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3717         } else {
3718             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3719         }
3720     }
3721
3722     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3723         push @$cmd, $fixups->@*;
3724     }
3725
3726     if ($conf->{vmgenid}) {
3727         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3728     }
3729
3730     # add usb controllers
3731     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3732         $conf, $bridges, $arch, $machine_type, $machine_version);
3733     push @$devices, @usbcontrollers if @usbcontrollers;
3734     my $vga = parse_vga($conf->{vga});
3735
3736     my $qxlnum = vga_conf_has_spice($conf->{vga});
3737     $vga->{type} = 'qxl' if $qxlnum;
3738
3739     if (!$vga->{type}) {
3740         if ($arch eq 'aarch64') {
3741             $vga->{type} = 'virtio';
3742         } elsif (min_version($machine_version, 2, 9)) {
3743             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3744         } else {
3745             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3746         }
3747     }
3748
3749     # enable absolute mouse coordinates (needed by vnc)
3750     my $tablet = $conf->{tablet};
3751     if (!defined($tablet)) {
3752         $tablet = $defaults->{tablet};
3753         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3754         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3755     }
3756
3757     if ($tablet) {
3758         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3759         my $kbd = print_keyboarddevice_full($conf, $arch);
3760         push @$devices, '-device', $kbd if defined($kbd);
3761     }
3762
3763     my $bootorder = device_bootorder($conf);
3764
3765     # host pci device passthrough
3766     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3767         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3768
3769     # usb devices
3770     my $usb_dev_features = {};
3771     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3772
3773     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3774         $conf, $usb_dev_features, $bootorder, $machine_version);
3775     push @$devices, @usbdevices if @usbdevices;
3776
3777     # serial devices
3778     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3779         my $path = $conf->{"serial$i"} or next;
3780         if ($path eq 'socket') {
3781             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3782             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3783             # On aarch64, serial0 is the UART device. QEMU only allows
3784             # connecting UART devices via the '-serial' command line, as
3785             # the device has a fixed slot on the hardware...
3786             if ($arch eq 'aarch64' && $i == 0) {
3787                 push @$devices, '-serial', "chardev:serial$i";
3788             } else {
3789                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3790             }
3791         } else {
3792             die "no such serial device\n" if ! -c $path;
3793             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3794             push @$devices, '-device', "isa-serial,chardev=serial$i";
3795         }
3796     }
3797
3798     # parallel devices
3799     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3800         if (my $path = $conf->{"parallel$i"}) {
3801             die "no such parallel device\n" if ! -c $path;
3802             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3803             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3804             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3805         }
3806     }
3807
3808     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3809         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3810         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3811         push @$devices, @$audio_devs;
3812     }
3813
3814     add_tpm_device($vmid, $devices, $conf);
3815
3816     my $sockets = 1;
3817     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3818     $sockets = $conf->{sockets} if  $conf->{sockets};
3819
3820     my $cores = $conf->{cores} || 1;
3821
3822     my $maxcpus = $sockets * $cores;
3823
3824     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3825
3826     my $allowed_vcpus = $cpuinfo->{cpus};
3827
3828     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3829
3830     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3831         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3832         for (my $i = 2; $i <= $vcpus; $i++)  {
3833             my $cpustr = print_cpu_device($conf,$i);
3834             push @$cmd, '-device', $cpustr;
3835         }
3836
3837     } else {
3838
3839         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3840     }
3841     push @$cmd, '-nodefaults';
3842
3843     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3844
3845     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3846
3847     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3848
3849     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3850         push @$devices, '-device', print_vga_device(
3851             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3852
3853         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3854
3855         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3856         push @$cmd,  '-vnc', "unix:$socket,password=on";
3857     } else {
3858         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3859         push @$cmd, '-nographic';
3860     }
3861
3862     # time drift fix
3863     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3864     my $useLocaltime = $conf->{localtime};
3865
3866     if ($winversion >= 5) { # windows
3867         $useLocaltime = 1 if !defined($conf->{localtime});
3868
3869         # use time drift fix when acpi is enabled
3870         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3871             $tdf = 1 if !defined($conf->{tdf});
3872         }
3873     }
3874
3875     if ($winversion >= 6) {
3876         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3877         push @$machineFlags, 'hpet=off';
3878     }
3879
3880     push @$rtcFlags, 'driftfix=slew' if $tdf;
3881
3882     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3883         push @$rtcFlags, "base=$conf->{startdate}";
3884     } elsif ($useLocaltime) {
3885         push @$rtcFlags, 'base=localtime';
3886     }
3887
3888     if ($forcecpu) {
3889         push @$cmd, '-cpu', $forcecpu;
3890     } else {
3891         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3892     }
3893
3894     PVE::QemuServer::Memory::config(
3895         $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory}, $cmd);
3896
3897     push @$cmd, '-S' if $conf->{freeze};
3898
3899     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3900
3901     my $guest_agent = parse_guest_agent($conf);
3902
3903     if ($guest_agent->{enabled}) {
3904         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3905         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3906
3907         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3908             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3909             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3910             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3911         } elsif ($guest_agent->{type} eq 'isa') {
3912             push @$devices, '-device', "isa-serial,chardev=qga0";
3913         }
3914     }
3915
3916     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3917     if ($rng && $version_guard->(4, 1, 2)) {
3918         check_rng_source($rng->{source});
3919
3920         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3921         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3922         my $limiter_str = "";
3923         if ($max_bytes) {
3924             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3925         }
3926
3927         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3928         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3929         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3930     }
3931
3932     my $spice_port;
3933
3934     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3935         if ($qxlnum > 1) {
3936             if ($winversion){
3937                 for (my $i = 1; $i < $qxlnum; $i++){
3938                     push @$devices, '-device', print_vga_device(
3939                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3940                 }
3941             } else {
3942                 # assume other OS works like Linux
3943                 my ($ram, $vram) = ("134217728", "67108864");
3944                 if ($vga->{memory}) {
3945                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3946                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3947                 }
3948                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3949                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3950             }
3951         }
3952
3953         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3954
3955         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3956         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3957         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3958
3959         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3960         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3961         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3962
3963         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3964         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3965
3966         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3967         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3968         if ($spice_enhancement->{foldersharing}) {
3969             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3970             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3971         }
3972
3973         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3974         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3975             if $spice_enhancement->{videostreaming};
3976
3977         push @$devices, '-spice', "$spice_opts";
3978     }
3979
3980     # enable balloon by default, unless explicitly disabled
3981     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3982         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3983         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3984         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3985         push @$devices, '-device', $ballooncmd;
3986     }
3987
3988     if ($conf->{watchdog}) {
3989         my $wdopts = parse_watchdog($conf->{watchdog});
3990         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3991         my $watchdog = $wdopts->{model} || 'i6300esb';
3992         push @$devices, '-device', "$watchdog$pciaddr";
3993         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3994     }
3995
3996     my $vollist = [];
3997     my $scsicontroller = {};
3998     my $ahcicontroller = {};
3999     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4000
4001     # Add iscsi initiator name if available
4002     if (my $initiator = get_initiator_name()) {
4003         push @$devices, '-iscsi', "initiator-name=$initiator";
4004     }
4005
4006     PVE::QemuConfig->foreach_volume($conf, sub {
4007         my ($ds, $drive) = @_;
4008
4009         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4010             check_volume_storage_type($storecfg, $drive->{file});
4011             push @$vollist, $drive->{file};
4012         }
4013
4014         # ignore efidisk here, already added in bios/fw handling code above
4015         return if $drive->{interface} eq 'efidisk';
4016         # similar for TPM
4017         return if $drive->{interface} eq 'tpmstate';
4018
4019         $use_virtio = 1 if $ds =~ m/^virtio/;
4020
4021         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4022
4023         if ($drive->{interface} eq 'virtio'){
4024            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4025         }
4026
4027         if ($drive->{interface} eq 'scsi') {
4028
4029             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4030
4031             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4032                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4033
4034             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4035             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4036
4037             my $iothread = '';
4038             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4039                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4040                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4041             } elsif ($drive->{iothread}) {
4042                 log_warn(
4043                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4044                 );
4045             }
4046
4047             my $queues = '';
4048             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4049                 $queues = ",num_queues=$drive->{queues}";
4050             }
4051
4052             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4053                 if !$scsicontroller->{$controller};
4054             $scsicontroller->{$controller}=1;
4055         }
4056
4057         if ($drive->{interface} eq 'sata') {
4058             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4059             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4060             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4061                 if !$ahcicontroller->{$controller};
4062             $ahcicontroller->{$controller}=1;
4063         }
4064
4065         my $pbs_conf = $pbs_backing->{$ds};
4066         my $pbs_name = undef;
4067         if ($pbs_conf) {
4068             $pbs_name = "drive-$ds-pbs";
4069             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4070         }
4071
4072         my $drive_cmd = print_drive_commandline_full(
4073             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4074
4075         # extra protection for templates, but SATA and IDE don't support it..
4076         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4077
4078         push @$devices, '-drive',$drive_cmd;
4079         push @$devices, '-device', print_drivedevice_full(
4080             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4081     });
4082
4083     for (my $i = 0; $i < $MAX_NETS; $i++) {
4084         my $netname = "net$i";
4085
4086         next if !$conf->{$netname};
4087         my $d = parse_net($conf->{$netname});
4088         next if !$d;
4089         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4090
4091         $use_virtio = 1 if $d->{model} eq 'virtio';
4092
4093         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4094
4095         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4096         push @$devices, '-netdev', $netdevfull;
4097
4098         my $netdevicefull = print_netdevice_full(
4099             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4100
4101         push @$devices, '-device', $netdevicefull;
4102     }
4103
4104     if ($conf->{ivshmem}) {
4105         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4106
4107         my $bus;
4108         if ($q35) {
4109             $bus = print_pcie_addr("ivshmem");
4110         } else {
4111             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4112         }
4113
4114         my $ivshmem_name = $ivshmem->{name} // $vmid;
4115         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4116
4117         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4118         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4119             .",size=$ivshmem->{size}M";
4120     }
4121
4122     # pci.4 is nested in pci.1
4123     $bridges->{1} = 1 if $bridges->{4};
4124
4125     if (!$q35) { # add pci bridges
4126         if (min_version($machine_version, 2, 3)) {
4127            $bridges->{1} = 1;
4128            $bridges->{2} = 1;
4129         }
4130         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4131     }
4132
4133     for my $k (sort {$b cmp $a} keys %$bridges) {
4134         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4135
4136         my $k_name = $k;
4137         if ($k == 2 && $legacy_igd) {
4138             $k_name = "$k-igd";
4139         }
4140         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4141         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4142
4143         if ($q35) { # add after -readconfig pve-q35.cfg
4144             splice @$devices, 2, 0, '-device', $devstr;
4145         } else {
4146             unshift @$devices, '-device', $devstr if $k > 0;
4147         }
4148     }
4149
4150     if (!$kvm) {
4151         push @$machineFlags, 'accel=tcg';
4152     }
4153
4154     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4155
4156     my $machine_type_min = $machine_type;
4157     if ($add_pve_version) {
4158         $machine_type_min =~ s/\+pve\d+$//;
4159         $machine_type_min .= "+pve$required_pve_version";
4160     }
4161     push @$machineFlags, "type=${machine_type_min}";
4162
4163     push @$cmd, @$devices;
4164     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4165     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4166     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4167
4168     if (my $vmstate = $conf->{vmstate}) {
4169         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4170         push @$vollist, $vmstate;
4171         push @$cmd, '-loadstate', $statepath;
4172         print "activating and using '$vmstate' as vmstate\n";
4173     }
4174
4175     if (PVE::QemuConfig->is_template($conf)) {
4176         # needed to workaround base volumes being read-only
4177         push @$cmd, '-snapshot';
4178     }
4179
4180     # add custom args
4181     if ($conf->{args}) {
4182         my $aa = PVE::Tools::split_args($conf->{args});
4183         push @$cmd, @$aa;
4184     }
4185
4186     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4187 }
4188
4189 sub check_rng_source {
4190     my ($source) = @_;
4191
4192     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4193     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4194         if ! -e $source;
4195
4196     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4197     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4198         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4199         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4200         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4201             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4202             ." to the host.\n";
4203     }
4204 }
4205
4206 sub spice_port {
4207     my ($vmid) = @_;
4208
4209     my $res = mon_cmd($vmid, 'query-spice');
4210
4211     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4212 }
4213
4214 sub vm_devices_list {
4215     my ($vmid) = @_;
4216
4217     my $res = mon_cmd($vmid, 'query-pci');
4218     my $devices_to_check = [];
4219     my $devices = {};
4220     foreach my $pcibus (@$res) {
4221         push @$devices_to_check, @{$pcibus->{devices}},
4222     }
4223
4224     while (@$devices_to_check) {
4225         my $to_check = [];
4226         for my $d (@$devices_to_check) {
4227             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4228             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4229
4230             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4231             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4232         }
4233         $devices_to_check = $to_check;
4234     }
4235
4236     my $resblock = mon_cmd($vmid, 'query-block');
4237     foreach my $block (@$resblock) {
4238         if($block->{device} =~ m/^drive-(\S+)/){
4239                 $devices->{$1} = 1;
4240         }
4241     }
4242
4243     my $resmice = mon_cmd($vmid, 'query-mice');
4244     foreach my $mice (@$resmice) {
4245         if ($mice->{name} eq 'QEMU HID Tablet') {
4246             $devices->{tablet} = 1;
4247             last;
4248         }
4249     }
4250
4251     # for usb devices there is no query-usb
4252     # but we can iterate over the entries in
4253     # qom-list path=/machine/peripheral
4254     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4255     foreach my $per (@$resperipheral) {
4256         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4257             $devices->{$per->{name}} = 1;
4258         }
4259     }
4260
4261     return $devices;
4262 }
4263
4264 sub vm_deviceplug {
4265     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4266
4267     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4268
4269     my $devices_list = vm_devices_list($vmid);
4270     return 1 if defined($devices_list->{$deviceid});
4271
4272     # add PCI bridge if we need it for the device
4273     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4274
4275     if ($deviceid eq 'tablet') {
4276         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4277     } elsif ($deviceid eq 'keyboard') {
4278         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4279     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4280         my $id = $1;
4281         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4282         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4283     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4284         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4285     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4286         qemu_iothread_add($vmid, $deviceid, $device);
4287
4288         qemu_driveadd($storecfg, $vmid, $device);
4289         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4290
4291         qemu_deviceadd($vmid, $devicefull);
4292         eval { qemu_deviceaddverify($vmid, $deviceid); };
4293         if (my $err = $@) {
4294             eval { qemu_drivedel($vmid, $deviceid); };
4295             warn $@ if $@;
4296             die $err;
4297         }
4298     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4299         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4300         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4301         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4302
4303         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4304
4305         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4306             qemu_iothread_add($vmid, $deviceid, $device);
4307             $devicefull .= ",iothread=iothread-$deviceid";
4308         }
4309
4310         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4311             $devicefull .= ",num_queues=$device->{queues}";
4312         }
4313
4314         qemu_deviceadd($vmid, $devicefull);
4315         qemu_deviceaddverify($vmid, $deviceid);
4316     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4317         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4318         qemu_driveadd($storecfg, $vmid, $device);
4319
4320         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4321         eval { qemu_deviceadd($vmid, $devicefull); };
4322         if (my $err = $@) {
4323             eval { qemu_drivedel($vmid, $deviceid); };
4324             warn $@ if $@;
4325             die $err;
4326         }
4327     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4328         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4329
4330         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4331         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4332         my $use_old_bios_files = undef;
4333         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4334
4335         my $netdevicefull = print_netdevice_full(
4336             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4337         qemu_deviceadd($vmid, $netdevicefull);
4338         eval {
4339             qemu_deviceaddverify($vmid, $deviceid);
4340             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4341         };
4342         if (my $err = $@) {
4343             eval { qemu_netdevdel($vmid, $deviceid); };
4344             warn $@ if $@;
4345             die $err;
4346         }
4347     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4348         my $bridgeid = $2;
4349         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4350         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4351
4352         qemu_deviceadd($vmid, $devicefull);
4353         qemu_deviceaddverify($vmid, $deviceid);
4354     } else {
4355         die "can't hotplug device '$deviceid'\n";
4356     }
4357
4358     return 1;
4359 }
4360
4361 # fixme: this should raise exceptions on error!
4362 sub vm_deviceunplug {
4363     my ($vmid, $conf, $deviceid) = @_;
4364
4365     my $devices_list = vm_devices_list($vmid);
4366     return 1 if !defined($devices_list->{$deviceid});
4367
4368     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4369     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4370
4371     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4372         qemu_devicedel($vmid, $deviceid);
4373     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4374         qemu_devicedel($vmid, $deviceid);
4375         qemu_devicedelverify($vmid, $deviceid);
4376     } elsif ($deviceid =~ m/^usb\d+$/) {
4377         qemu_devicedel($vmid, $deviceid);
4378         qemu_devicedelverify($vmid, $deviceid);
4379     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4380         my $device = parse_drive($deviceid, $conf->{$deviceid});
4381
4382         qemu_devicedel($vmid, $deviceid);
4383         qemu_devicedelverify($vmid, $deviceid);
4384         qemu_drivedel($vmid, $deviceid);
4385         qemu_iothread_del($vmid, $deviceid, $device);
4386     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4387         qemu_devicedel($vmid, $deviceid);
4388         qemu_devicedelverify($vmid, $deviceid);
4389     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4390         my $device = parse_drive($deviceid, $conf->{$deviceid});
4391
4392         qemu_devicedel($vmid, $deviceid);
4393         qemu_devicedelverify($vmid, $deviceid);
4394         qemu_drivedel($vmid, $deviceid);
4395         qemu_deletescsihw($conf, $vmid, $deviceid);
4396
4397         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4398             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4399     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4400         qemu_devicedel($vmid, $deviceid);
4401         qemu_devicedelverify($vmid, $deviceid);
4402         qemu_netdevdel($vmid, $deviceid);
4403     } else {
4404         die "can't unplug device '$deviceid'\n";
4405     }
4406
4407     return 1;
4408 }
4409
4410 sub qemu_spice_usbredir_chardev_add {
4411     my ($vmid, $id) = @_;
4412
4413     mon_cmd($vmid, "chardev-add" , (
4414         id => $id,
4415         backend => {
4416             type => 'spicevmc',
4417             data => {
4418                 type => "usbredir",
4419             },
4420         },
4421     ));
4422 }
4423
4424 sub qemu_deviceadd {
4425     my ($vmid, $devicefull) = @_;
4426
4427     $devicefull = "driver=".$devicefull;
4428     my %options =  split(/[=,]/, $devicefull);
4429
4430     mon_cmd($vmid, "device_add" , %options);
4431 }
4432
4433 sub qemu_devicedel {
4434     my ($vmid, $deviceid) = @_;
4435
4436     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4437 }
4438
4439 sub qemu_iothread_add {
4440     my ($vmid, $deviceid, $device) = @_;
4441
4442     if ($device->{iothread}) {
4443         my $iothreads = vm_iothreads_list($vmid);
4444         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4445     }
4446 }
4447
4448 sub qemu_iothread_del {
4449     my ($vmid, $deviceid, $device) = @_;
4450
4451     if ($device->{iothread}) {
4452         my $iothreads = vm_iothreads_list($vmid);
4453         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4454     }
4455 }
4456
4457 sub qemu_objectadd {
4458     my ($vmid, $objectid, $qomtype) = @_;
4459
4460     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4461
4462     return 1;
4463 }
4464
4465 sub qemu_objectdel {
4466     my ($vmid, $objectid) = @_;
4467
4468     mon_cmd($vmid, "object-del", id => $objectid);
4469
4470     return 1;
4471 }
4472
4473 sub qemu_driveadd {
4474     my ($storecfg, $vmid, $device) = @_;
4475
4476     my $kvmver = get_running_qemu_version($vmid);
4477     my $io_uring = min_version($kvmver, 6, 0);
4478     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4479     $drive =~ s/\\/\\\\/g;
4480     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4481
4482     # If the command succeeds qemu prints: "OK"
4483     return 1 if $ret =~ m/OK/s;
4484
4485     die "adding drive failed: $ret\n";
4486 }
4487
4488 sub qemu_drivedel {
4489     my ($vmid, $deviceid) = @_;
4490
4491     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4492     $ret =~ s/^\s+//;
4493
4494     return 1 if $ret eq "";
4495
4496     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4497     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4498
4499     die "deleting drive $deviceid failed : $ret\n";
4500 }
4501
4502 sub qemu_deviceaddverify {
4503     my ($vmid, $deviceid) = @_;
4504
4505     for (my $i = 0; $i <= 5; $i++) {
4506          my $devices_list = vm_devices_list($vmid);
4507          return 1 if defined($devices_list->{$deviceid});
4508          sleep 1;
4509     }
4510
4511     die "error on hotplug device '$deviceid'\n";
4512 }
4513
4514
4515 sub qemu_devicedelverify {
4516     my ($vmid, $deviceid) = @_;
4517
4518     # need to verify that the device is correctly removed as device_del
4519     # is async and empty return is not reliable
4520
4521     for (my $i = 0; $i <= 5; $i++) {
4522          my $devices_list = vm_devices_list($vmid);
4523          return 1 if !defined($devices_list->{$deviceid});
4524          sleep 1;
4525     }
4526
4527     die "error on hot-unplugging device '$deviceid'\n";
4528 }
4529
4530 sub qemu_findorcreatescsihw {
4531     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4532
4533     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4534
4535     my $scsihwid="$controller_prefix$controller";
4536     my $devices_list = vm_devices_list($vmid);
4537
4538     if (!defined($devices_list->{$scsihwid})) {
4539         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4540     }
4541
4542     return 1;
4543 }
4544
4545 sub qemu_deletescsihw {
4546     my ($conf, $vmid, $opt) = @_;
4547
4548     my $device = parse_drive($opt, $conf->{$opt});
4549
4550     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4551         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4552         return 1;
4553     }
4554
4555     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4556
4557     my $devices_list = vm_devices_list($vmid);
4558     foreach my $opt (keys %{$devices_list}) {
4559         if (is_valid_drivename($opt)) {
4560             my $drive = parse_drive($opt, $conf->{$opt});
4561             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4562                 return 1;
4563             }
4564         }
4565     }
4566
4567     my $scsihwid="scsihw$controller";
4568
4569     vm_deviceunplug($vmid, $conf, $scsihwid);
4570
4571     return 1;
4572 }
4573
4574 sub qemu_add_pci_bridge {
4575     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4576
4577     my $bridges = {};
4578
4579     my $bridgeid;
4580
4581     print_pci_addr($device, $bridges, $arch, $machine_type);
4582
4583     while (my ($k, $v) = each %$bridges) {
4584         $bridgeid = $k;
4585     }
4586     return 1 if !defined($bridgeid) || $bridgeid < 1;
4587
4588     my $bridge = "pci.$bridgeid";
4589     my $devices_list = vm_devices_list($vmid);
4590
4591     if (!defined($devices_list->{$bridge})) {
4592         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4593     }
4594
4595     return 1;
4596 }
4597
4598 sub qemu_set_link_status {
4599     my ($vmid, $device, $up) = @_;
4600
4601     mon_cmd($vmid, "set_link", name => $device,
4602                up => $up ? JSON::true : JSON::false);
4603 }
4604
4605 sub qemu_netdevadd {
4606     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4607
4608     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4609     my %options =  split(/[=,]/, $netdev);
4610
4611     if (defined(my $vhost = $options{vhost})) {
4612         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4613     }
4614
4615     if (defined(my $queues = $options{queues})) {
4616         $options{queues} = $queues + 0;
4617     }
4618
4619     mon_cmd($vmid, "netdev_add",  %options);
4620     return 1;
4621 }
4622
4623 sub qemu_netdevdel {
4624     my ($vmid, $deviceid) = @_;
4625
4626     mon_cmd($vmid, "netdev_del", id => $deviceid);
4627 }
4628
4629 sub qemu_usb_hotplug {
4630     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4631
4632     return if !$device;
4633
4634     # remove the old one first
4635     vm_deviceunplug($vmid, $conf, $deviceid);
4636
4637     # check if xhci controller is necessary and available
4638     my $devicelist = vm_devices_list($vmid);
4639
4640     if (!$devicelist->{xhci}) {
4641         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4642         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4643     }
4644
4645     # add the new one
4646     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4647 }
4648
4649 sub qemu_cpu_hotplug {
4650     my ($vmid, $conf, $vcpus) = @_;
4651
4652     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4653
4654     my $sockets = 1;
4655     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4656     $sockets = $conf->{sockets} if  $conf->{sockets};
4657     my $cores = $conf->{cores} || 1;
4658     my $maxcpus = $sockets * $cores;
4659
4660     $vcpus = $maxcpus if !$vcpus;
4661
4662     die "you can't add more vcpus than maxcpus\n"
4663         if $vcpus > $maxcpus;
4664
4665     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4666
4667     if ($vcpus < $currentvcpus) {
4668
4669         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4670
4671             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4672                 qemu_devicedel($vmid, "cpu$i");
4673                 my $retry = 0;
4674                 my $currentrunningvcpus = undef;
4675                 while (1) {
4676                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4677                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4678                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4679                     $retry++;
4680                     sleep 1;
4681                 }
4682                 #update conf after each succesfull cpu unplug
4683                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4684                 PVE::QemuConfig->write_config($vmid, $conf);
4685             }
4686         } else {
4687             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4688         }
4689
4690         return;
4691     }
4692
4693     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4694     die "vcpus in running vm does not match its configuration\n"
4695         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4696
4697     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4698
4699         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4700             my $cpustr = print_cpu_device($conf, $i);
4701             qemu_deviceadd($vmid, $cpustr);
4702
4703             my $retry = 0;
4704             my $currentrunningvcpus = undef;
4705             while (1) {
4706                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4707                 last if scalar(@{$currentrunningvcpus}) == $i;
4708                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4709                 sleep 1;
4710                 $retry++;
4711             }
4712             #update conf after each succesfull cpu hotplug
4713             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4714             PVE::QemuConfig->write_config($vmid, $conf);
4715         }
4716     } else {
4717
4718         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4719             mon_cmd($vmid, "cpu-add", id => int($i));
4720         }
4721     }
4722 }
4723
4724 sub qemu_block_set_io_throttle {
4725     my ($vmid, $deviceid,
4726         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4727         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4728         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4729         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4730
4731     return if !check_running($vmid) ;
4732
4733     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4734         bps => int($bps),
4735         bps_rd => int($bps_rd),
4736         bps_wr => int($bps_wr),
4737         iops => int($iops),
4738         iops_rd => int($iops_rd),
4739         iops_wr => int($iops_wr),
4740         bps_max => int($bps_max),
4741         bps_rd_max => int($bps_rd_max),
4742         bps_wr_max => int($bps_wr_max),
4743         iops_max => int($iops_max),
4744         iops_rd_max => int($iops_rd_max),
4745         iops_wr_max => int($iops_wr_max),
4746         bps_max_length => int($bps_max_length),
4747         bps_rd_max_length => int($bps_rd_max_length),
4748         bps_wr_max_length => int($bps_wr_max_length),
4749         iops_max_length => int($iops_max_length),
4750         iops_rd_max_length => int($iops_rd_max_length),
4751         iops_wr_max_length => int($iops_wr_max_length),
4752     );
4753
4754 }
4755
4756 sub qemu_block_resize {
4757     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4758
4759     my $running = check_running($vmid);
4760
4761     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4762
4763     return if !$running;
4764
4765     my $padding = (1024 - $size % 1024) % 1024;
4766     $size = $size + $padding;
4767
4768     mon_cmd(
4769         $vmid,
4770         "block_resize",
4771         device => $deviceid,
4772         size => int($size),
4773         timeout => 60,
4774     );
4775 }
4776
4777 sub qemu_volume_snapshot {
4778     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4779
4780     my $running = check_running($vmid);
4781
4782     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4783         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4784     } else {
4785         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4786     }
4787 }
4788
4789 sub qemu_volume_snapshot_delete {
4790     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4791
4792     my $running = check_running($vmid);
4793
4794     if($running) {
4795
4796         $running = undef;
4797         my $conf = PVE::QemuConfig->load_config($vmid);
4798         PVE::QemuConfig->foreach_volume($conf, sub {
4799             my ($ds, $drive) = @_;
4800             $running = 1 if $drive->{file} eq $volid;
4801         });
4802     }
4803
4804     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4805         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4806     } else {
4807         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4808     }
4809 }
4810
4811 sub set_migration_caps {
4812     my ($vmid, $savevm) = @_;
4813
4814     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4815
4816     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4817     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4818
4819     my $cap_ref = [];
4820
4821     my $enabled_cap = {
4822         "auto-converge" => 1,
4823         "xbzrle" => 1,
4824         "x-rdma-pin-all" => 0,
4825         "zero-blocks" => 0,
4826         "compress" => 0,
4827         "dirty-bitmaps" => $dirty_bitmaps,
4828     };
4829
4830     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4831
4832     for my $supported_capability (@$supported_capabilities) {
4833         push @$cap_ref, {
4834             capability => $supported_capability->{capability},
4835             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4836         };
4837     }
4838
4839     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4840 }
4841
4842 sub foreach_volid {
4843     my ($conf, $include_pending, $func, @param) = @_;
4844
4845     my $volhash = {};
4846
4847     my $test_volid = sub {
4848         my ($key, $drive, $snapname, $pending) = @_;
4849
4850         my $volid = $drive->{file};
4851         return if !$volid;
4852
4853         $volhash->{$volid}->{cdrom} //= 1;
4854         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4855
4856         my $replicate = $drive->{replicate} // 1;
4857         $volhash->{$volid}->{replicate} //= 0;
4858         $volhash->{$volid}->{replicate} = 1 if $replicate;
4859
4860         $volhash->{$volid}->{shared} //= 0;
4861         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4862
4863         $volhash->{$volid}->{referenced_in_config} //= 0;
4864         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname) && !defined($pending);
4865
4866         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4867             if defined($snapname);
4868
4869         $volhash->{$volid}->{referenced_in_pending} = 1 if defined($pending);
4870
4871         my $size = $drive->{size};
4872         $volhash->{$volid}->{size} //= $size if $size;
4873
4874         $volhash->{$volid}->{is_vmstate} //= 0;
4875         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4876
4877         $volhash->{$volid}->{is_tpmstate} //= 0;
4878         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4879
4880         $volhash->{$volid}->{is_unused} //= 0;
4881         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4882
4883         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4884     };
4885
4886     my $include_opts = {
4887         extra_keys => ['vmstate'],
4888         include_unused => 1,
4889     };
4890
4891     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4892
4893     if ($include_pending && defined($conf->{pending}) && $conf->{pending}->%*) {
4894         PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1);
4895     }
4896
4897     foreach my $snapname (keys %{$conf->{snapshots}}) {
4898         my $snap = $conf->{snapshots}->{$snapname};
4899         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4900     }
4901
4902     foreach my $volid (keys %$volhash) {
4903         &$func($volid, $volhash->{$volid}, @param);
4904     }
4905 }
4906
4907 my $fast_plug_option = {
4908     'description' => 1,
4909     'hookscript' => 1,
4910     'lock' => 1,
4911     'migrate_downtime' => 1,
4912     'migrate_speed' => 1,
4913     'name' => 1,
4914     'onboot' => 1,
4915     'protection' => 1,
4916     'shares' => 1,
4917     'startup' => 1,
4918     'tags' => 1,
4919     'vmstatestorage' => 1,
4920 };
4921
4922 for my $opt (keys %$confdesc_cloudinit) {
4923     $fast_plug_option->{$opt} = 1;
4924 };
4925
4926 # hotplug changes in [PENDING]
4927 # $selection hash can be used to only apply specified options, for
4928 # example: { cores => 1 } (only apply changed 'cores')
4929 # $errors ref is used to return error messages
4930 sub vmconfig_hotplug_pending {
4931     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4932
4933     my $defaults = load_defaults();
4934     my $arch = get_vm_arch($conf);
4935     my $machine_type = get_vm_machine($conf, undef, $arch);
4936
4937     # commit values which do not have any impact on running VM first
4938     # Note: those option cannot raise errors, we we do not care about
4939     # $selection and always apply them.
4940
4941     my $add_error = sub {
4942         my ($opt, $msg) = @_;
4943         $errors->{$opt} = "hotplug problem - $msg";
4944     };
4945
4946     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4947
4948     my $cloudinit_record_changed = sub {
4949         my ($conf, $opt, $old, $new) = @_;
4950         return if !$cloudinit_pending_properties->{$opt};
4951
4952         my $ci = ($conf->{cloudinit} //= {});
4953
4954         my $recorded = $ci->{$opt};
4955         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4956
4957         if (defined($new)) {
4958             if (defined($old)) {
4959                 # an existing value is being modified
4960                 if (defined($recorded)) {
4961                     # the value was already not in sync
4962                     if ($new eq $recorded) {
4963                         # a value is being reverted to the cloud-init state:
4964                         delete $ci->{$opt};
4965                         delete $added{$opt};
4966                     } else {
4967                         # the value was changed multiple times, do nothing
4968                     }
4969                 } elsif ($added{$opt}) {
4970                     # the value had been marked as added and is being changed, do nothing
4971                 } else {
4972                     # the value is new, record it:
4973                     $ci->{$opt} = $old;
4974                 }
4975             } else {
4976                 # a new value is being added
4977                 if (defined($recorded)) {
4978                     # it was already not in sync
4979                     if ($new eq $recorded) {
4980                         # a value is being reverted to the cloud-init state:
4981                         delete $ci->{$opt};
4982                         delete $added{$opt};
4983                     } else {
4984                         # the value had temporarily been removed, do nothing
4985                     }
4986                 } elsif ($added{$opt}) {
4987                     # the value had been marked as added already, do nothing
4988                 } else {
4989                     # the value is new, add it
4990                     $added{$opt} = 1;
4991                 }
4992             }
4993         } elsif (!defined($old)) {
4994             # a non-existent value is being removed? ignore...
4995         } else {
4996             # a value is being deleted
4997             if (defined($recorded)) {
4998                 # a value was already recorded, just keep it
4999             } elsif ($added{$opt}) {
5000                 # the value was marked as added, remove it
5001                 delete $added{$opt};
5002             } else {
5003                 # a previously unrecorded value is being removed, record the old value:
5004                 $ci->{$opt} = $old;
5005             }
5006         }
5007
5008         my $added = join(',', sort keys %added);
5009         $ci->{added} = $added if length($added);
5010     };
5011
5012     my $changes = 0;
5013     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5014         if ($fast_plug_option->{$opt}) {
5015             my $new = delete $conf->{pending}->{$opt};
5016             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
5017             $conf->{$opt} = $new;
5018             $changes = 1;
5019         }
5020     }
5021
5022     if ($changes) {
5023         PVE::QemuConfig->write_config($vmid, $conf);
5024     }
5025
5026     my $ostype = $conf->{ostype};
5027     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5028     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5029     my $usb_hotplug = $hotplug_features->{usb}
5030         && min_version($version, 7, 1)
5031         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5032
5033     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5034     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5035
5036     foreach my $opt (sort keys %$pending_delete_hash) {
5037         next if $selection && !$selection->{$opt};
5038         my $force = $pending_delete_hash->{$opt}->{force};
5039         eval {
5040             if ($opt eq 'hotplug') {
5041                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5042             } elsif ($opt eq 'tablet') {
5043                 die "skip\n" if !$hotplug_features->{usb};
5044                 if ($defaults->{tablet}) {
5045                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5046                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5047                         if $arch eq 'aarch64';
5048                 } else {
5049                     vm_deviceunplug($vmid, $conf, 'tablet');
5050                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5051                 }
5052             } elsif ($opt =~ m/^usb(\d+)$/) {
5053                 my $index = $1;
5054                 die "skip\n" if !$usb_hotplug;
5055                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5056                 vm_deviceunplug($vmid, $conf, $opt);
5057             } elsif ($opt eq 'vcpus') {
5058                 die "skip\n" if !$hotplug_features->{cpu};
5059                 qemu_cpu_hotplug($vmid, $conf, undef);
5060             } elsif ($opt eq 'balloon') {
5061                 # enable balloon device is not hotpluggable
5062                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5063                 # here we reset the ballooning value to memory
5064                 my $balloon = $conf->{memory} || $defaults->{memory};
5065                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5066             } elsif ($fast_plug_option->{$opt}) {
5067                 # do nothing
5068             } elsif ($opt =~ m/^net(\d+)$/) {
5069                 die "skip\n" if !$hotplug_features->{network};
5070                 vm_deviceunplug($vmid, $conf, $opt);
5071             } elsif (is_valid_drivename($opt)) {
5072                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5073                 vm_deviceunplug($vmid, $conf, $opt);
5074                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5075             } elsif ($opt =~ m/^memory$/) {
5076                 die "skip\n" if !$hotplug_features->{memory};
5077                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5078             } elsif ($opt eq 'cpuunits') {
5079                 $cgroup->change_cpu_shares(undef);
5080             } elsif ($opt eq 'cpulimit') {
5081                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5082             } else {
5083                 die "skip\n";
5084             }
5085         };
5086         if (my $err = $@) {
5087             &$add_error($opt, $err) if $err ne "skip\n";
5088         } else {
5089             my $old = delete $conf->{$opt};
5090             $cloudinit_record_changed->($conf, $opt, $old, undef);
5091             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5092         }
5093     }
5094
5095     my $cloudinit_opt;
5096     foreach my $opt (keys %{$conf->{pending}}) {
5097         next if $selection && !$selection->{$opt};
5098         my $value = $conf->{pending}->{$opt};
5099         eval {
5100             if ($opt eq 'hotplug') {
5101                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5102             } elsif ($opt eq 'tablet') {
5103                 die "skip\n" if !$hotplug_features->{usb};
5104                 if ($value == 1) {
5105                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5106                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5107                         if $arch eq 'aarch64';
5108                 } elsif ($value == 0) {
5109                     vm_deviceunplug($vmid, $conf, 'tablet');
5110                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5111                 }
5112             } elsif ($opt =~ m/^usb(\d+)$/) {
5113                 my $index = $1;
5114                 die "skip\n" if !$usb_hotplug;
5115                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5116                 my $id = $opt;
5117                 if ($d->{host} =~ m/^spice$/i)  {
5118                     $id = "usbredirdev$index";
5119                 }
5120                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5121             } elsif ($opt eq 'vcpus') {
5122                 die "skip\n" if !$hotplug_features->{cpu};
5123                 qemu_cpu_hotplug($vmid, $conf, $value);
5124             } elsif ($opt eq 'balloon') {
5125                 # enable/disable balloning device is not hotpluggable
5126                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5127                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5128                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5129
5130                 # allow manual ballooning if shares is set to zero
5131                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5132                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5133                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5134                 }
5135             } elsif ($opt =~ m/^net(\d+)$/) {
5136                 # some changes can be done without hotplug
5137                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5138                                     $vmid, $opt, $value, $arch, $machine_type);
5139             } elsif (is_valid_drivename($opt)) {
5140                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5141                 # some changes can be done without hotplug
5142                 my $drive = parse_drive($opt, $value);
5143                 if (drive_is_cloudinit($drive)) {
5144                     $cloudinit_opt = [$opt, $drive];
5145                     # apply all the other changes first, then generate the cloudinit disk
5146                     die "skip\n";
5147                 }
5148                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5149                                      $vmid, $opt, $value, $arch, $machine_type);
5150             } elsif ($opt =~ m/^memory$/) { #dimms
5151                 die "skip\n" if !$hotplug_features->{memory};
5152                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5153             } elsif ($opt eq 'cpuunits') {
5154                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5155                 $cgroup->change_cpu_shares($new_cpuunits);
5156             } elsif ($opt eq 'cpulimit') {
5157                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5158                 $cgroup->change_cpu_quota($cpulimit, 100000);
5159             } elsif ($opt eq 'agent') {
5160                 vmconfig_update_agent($conf, $opt, $value);
5161             } else {
5162                 die "skip\n";  # skip non-hot-pluggable options
5163             }
5164         };
5165         if (my $err = $@) {
5166             &$add_error($opt, $err) if $err ne "skip\n";
5167         } else {
5168             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5169             $conf->{$opt} = $value;
5170             delete $conf->{pending}->{$opt};
5171         }
5172     }
5173
5174     if (defined($cloudinit_opt)) {
5175         my ($opt, $drive) = @$cloudinit_opt;
5176         my $value = $conf->{pending}->{$opt};
5177         eval {
5178             my $temp = {%$conf, $opt => $value};
5179             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5180             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5181                                  $vmid, $opt, $value, $arch, $machine_type);
5182         };
5183         if (my $err = $@) {
5184             &$add_error($opt, $err) if $err ne "skip\n";
5185         } else {
5186             $conf->{$opt} = $value;
5187             delete $conf->{pending}->{$opt};
5188         }
5189     }
5190
5191     # unplug xhci controller if no usb device is left
5192     if ($usb_hotplug) {
5193         my $has_usb = 0;
5194         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5195             next if !defined($conf->{"usb$i"});
5196             $has_usb = 1;
5197             last;
5198         }
5199         if (!$has_usb) {
5200             vm_deviceunplug($vmid, $conf, 'xhci');
5201         }
5202     }
5203
5204     PVE::QemuConfig->write_config($vmid, $conf);
5205
5206     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5207         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5208     }
5209 }
5210
5211 sub try_deallocate_drive {
5212     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5213
5214     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5215         my $volid = $drive->{file};
5216         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5217             my $sid = PVE::Storage::parse_volume_id($volid);
5218             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5219
5220             # check if the disk is really unused
5221             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5222                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5223             PVE::Storage::vdisk_free($storecfg, $volid);
5224             return 1;
5225         } else {
5226             # If vm is not owner of this disk remove from config
5227             return 1;
5228         }
5229     }
5230
5231     return;
5232 }
5233
5234 sub vmconfig_delete_or_detach_drive {
5235     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5236
5237     my $drive = parse_drive($opt, $conf->{$opt});
5238
5239     my $rpcenv = PVE::RPCEnvironment::get();
5240     my $authuser = $rpcenv->get_user();
5241
5242     if ($force) {
5243         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5244         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5245     } else {
5246         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5247     }
5248 }
5249
5250
5251
5252 sub vmconfig_apply_pending {
5253     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5254
5255     return if !scalar(keys %{$conf->{pending}});
5256
5257     my $add_apply_error = sub {
5258         my ($opt, $msg) = @_;
5259         my $err_msg = "unable to apply pending change $opt : $msg";
5260         $errors->{$opt} = $err_msg;
5261         warn $err_msg;
5262     };
5263
5264     # cold plug
5265
5266     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5267     foreach my $opt (sort keys %$pending_delete_hash) {
5268         my $force = $pending_delete_hash->{$opt}->{force};
5269         eval {
5270             if ($opt =~ m/^unused/) {
5271                 die "internal error";
5272             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5273                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5274             }
5275         };
5276         if (my $err = $@) {
5277             $add_apply_error->($opt, $err);
5278         } else {
5279             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5280             delete $conf->{$opt};
5281         }
5282     }
5283
5284     PVE::QemuConfig->cleanup_pending($conf);
5285
5286     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5287
5288     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5289         next if $opt eq 'delete'; # just to be sure
5290         eval {
5291             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5292                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5293             }
5294         };
5295         if (my $err = $@) {
5296             $add_apply_error->($opt, $err);
5297         } else {
5298
5299             if (is_valid_drivename($opt)) {
5300                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5301                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5302             }
5303
5304             $conf->{$opt} = delete $conf->{pending}->{$opt};
5305         }
5306     }
5307
5308     # write all changes at once to avoid unnecessary i/o
5309     PVE::QemuConfig->write_config($vmid, $conf);
5310     if ($generate_cloudinit) {
5311         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5312             # After successful generation and if there were changes to be applied, update the
5313             # config to drop the {cloudinit} entry.
5314             PVE::QemuConfig->write_config($vmid, $conf);
5315         }
5316     }
5317 }
5318
5319 sub vmconfig_update_net {
5320     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5321
5322     my $newnet = parse_net($value);
5323
5324     if ($conf->{$opt}) {
5325         my $oldnet = parse_net($conf->{$opt});
5326
5327         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5328             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5329             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5330             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5331             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5332
5333             # for non online change, we try to hot-unplug
5334             die "skip\n" if !$hotplug;
5335             vm_deviceunplug($vmid, $conf, $opt);
5336         } else {
5337
5338             die "internal error" if $opt !~ m/net(\d+)/;
5339             my $iface = "tap${vmid}i$1";
5340
5341             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5342                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5343                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5344                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5345                 PVE::Network::tap_unplug($iface);
5346
5347                 if ($have_sdn) {
5348                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5349                 } else {
5350                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5351                 }
5352             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5353                 # Rate can be applied on its own but any change above needs to
5354                 # include the rate in tap_plug since OVS resets everything.
5355                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5356             }
5357
5358             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5359                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5360             }
5361
5362             return 1;
5363         }
5364     }
5365
5366     if ($hotplug) {
5367         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5368     } else {
5369         die "skip\n";
5370     }
5371 }
5372
5373 sub vmconfig_update_agent {
5374     my ($conf, $opt, $value) = @_;
5375
5376     die "skip\n" if !$conf->{$opt};
5377
5378     my $hotplug_options = { fstrim_cloned_disks => 1 };
5379
5380     my $old_agent = parse_guest_agent($conf);
5381     my $agent = parse_guest_agent({$opt => $value});
5382
5383     for my $option (keys %$agent) { # added/changed options
5384         next if defined($hotplug_options->{$option});
5385         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5386     }
5387
5388     for my $option (keys %$old_agent) { # removed options
5389         next if defined($hotplug_options->{$option});
5390         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5391     }
5392
5393     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5394 }
5395
5396 sub vmconfig_update_disk {
5397     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5398
5399     my $drive = parse_drive($opt, $value);
5400
5401     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5402         my $media = $drive->{media} || 'disk';
5403         my $oldmedia = $old_drive->{media} || 'disk';
5404         die "unable to change media type\n" if $media ne $oldmedia;
5405
5406         if (!drive_is_cdrom($old_drive)) {
5407
5408             if ($drive->{file} ne $old_drive->{file}) {
5409
5410                 die "skip\n" if !$hotplug;
5411
5412                 # unplug and register as unused
5413                 vm_deviceunplug($vmid, $conf, $opt);
5414                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5415
5416             } else {
5417                 # update existing disk
5418
5419                 # skip non hotpluggable value
5420                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5421                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5422                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5423                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5424                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5425                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5426                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5427                     die "skip\n";
5428                 }
5429
5430                 # apply throttle
5431                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5432                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5433                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5434                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5435                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5436                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5437                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5438                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5439                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5440                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5441                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5442                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5443                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5444                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5445                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5446                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5447                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5448                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5449
5450                     qemu_block_set_io_throttle(
5451                         $vmid,"drive-$opt",
5452                         ($drive->{mbps} || 0)*1024*1024,
5453                         ($drive->{mbps_rd} || 0)*1024*1024,
5454                         ($drive->{mbps_wr} || 0)*1024*1024,
5455                          $drive->{iops} || 0,
5456                          $drive->{iops_rd} || 0,
5457                          $drive->{iops_wr} || 0,
5458                         ($drive->{mbps_max} || 0)*1024*1024,
5459                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5460                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5461                          $drive->{iops_max} || 0,
5462                          $drive->{iops_rd_max} || 0,
5463                          $drive->{iops_wr_max} || 0,
5464                          $drive->{bps_max_length} || 1,
5465                          $drive->{bps_rd_max_length} || 1,
5466                          $drive->{bps_wr_max_length} || 1,
5467                          $drive->{iops_max_length} || 1,
5468                          $drive->{iops_rd_max_length} || 1,
5469                          $drive->{iops_wr_max_length} || 1,
5470                     );
5471
5472                 }
5473
5474                 return 1;
5475             }
5476
5477         } else { # cdrom
5478
5479             if ($drive->{file} eq 'none') {
5480                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5481                 if (drive_is_cloudinit($old_drive)) {
5482                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5483                 }
5484             } else {
5485                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5486
5487                 # force eject if locked
5488                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5489
5490                 if ($path) {
5491                     mon_cmd($vmid, "blockdev-change-medium",
5492                         id => "$opt", filename => "$path");
5493                 }
5494             }
5495
5496             return 1;
5497         }
5498     }
5499
5500     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5501     # hotplug new disks
5502     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5503     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5504 }
5505
5506 sub vmconfig_update_cloudinit_drive {
5507     my ($storecfg, $conf, $vmid) = @_;
5508
5509     my $cloudinit_ds = undef;
5510     my $cloudinit_drive = undef;
5511
5512     PVE::QemuConfig->foreach_volume($conf, sub {
5513         my ($ds, $drive) = @_;
5514         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5515             $cloudinit_ds = $ds;
5516             $cloudinit_drive = $drive;
5517         }
5518     });
5519
5520     return if !$cloudinit_drive;
5521
5522     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5523         PVE::QemuConfig->write_config($vmid, $conf);
5524     }
5525
5526     my $running = PVE::QemuServer::check_running($vmid);
5527
5528     if ($running) {
5529         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5530         if ($path) {
5531             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5532             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5533         }
5534     }
5535 }
5536
5537 # called in locked context by incoming migration
5538 sub vm_migrate_get_nbd_disks {
5539     my ($storecfg, $conf, $replicated_volumes) = @_;
5540
5541     my $local_volumes = {};
5542     PVE::QemuConfig->foreach_volume($conf, sub {
5543         my ($ds, $drive) = @_;
5544
5545         return if drive_is_cdrom($drive);
5546         return if $ds eq 'tpmstate0';
5547
5548         my $volid = $drive->{file};
5549
5550         return if !$volid;
5551
5552         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5553
5554         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5555         return if $scfg->{shared};
5556
5557         # replicated disks re-use existing state via bitmap
5558         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5559         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5560     });
5561     return $local_volumes;
5562 }
5563
5564 # called in locked context by incoming migration
5565 sub vm_migrate_alloc_nbd_disks {
5566     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5567
5568     my $nbd = {};
5569     foreach my $opt (sort keys %$source_volumes) {
5570         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5571
5572         if ($use_existing) {
5573             $nbd->{$opt}->{drivestr} = print_drive($drive);
5574             $nbd->{$opt}->{volid} = $volid;
5575             $nbd->{$opt}->{replicated} = 1;
5576             next;
5577         }
5578
5579         # storage mapping + volname = regular migration
5580         # storage mapping + format = remote migration
5581         # order of precedence, filtered by whether storage supports it:
5582         # 1. explicit requested format
5583         # 2. format of current volume
5584         # 3. default format of storage
5585         if (!$storagemap->{identity}) {
5586             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5587             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5588             if (!$format || !grep { $format eq $_ } @$validFormats) {
5589                 if ($volname) {
5590                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5591                     my $fileFormat = qemu_img_format($scfg, $volname);
5592                     $format = $fileFormat
5593                         if grep { $fileFormat eq $_ } @$validFormats;
5594                 }
5595                 $format //= $defFormat;
5596             }
5597         } else {
5598             # can't happen for remote migration, so $volname is always defined
5599             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5600             $format = qemu_img_format($scfg, $volname);
5601         }
5602
5603         my $size = $drive->{size} / 1024;
5604         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5605         my $newdrive = $drive;
5606         $newdrive->{format} = $format;
5607         $newdrive->{file} = $newvolid;
5608         my $drivestr = print_drive($newdrive);
5609         $nbd->{$opt}->{drivestr} = $drivestr;
5610         $nbd->{$opt}->{volid} = $newvolid;
5611     }
5612
5613     return $nbd;
5614 }
5615
5616 # see vm_start_nolock for parameters, additionally:
5617 # migrate_opts:
5618 #   storagemap = parsed storage map for allocating NBD disks
5619 sub vm_start {
5620     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5621
5622     return PVE::QemuConfig->lock_config($vmid, sub {
5623         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5624
5625         die "you can't start a vm if it's a template\n"
5626             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5627
5628         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5629         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5630
5631         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5632
5633         if ($has_backup_lock && $running) {
5634             # a backup is currently running, attempt to start the guest in the
5635             # existing QEMU instance
5636             return vm_resume($vmid);
5637         }
5638
5639         PVE::QemuConfig->check_lock($conf)
5640             if !($params->{skiplock} || $has_suspended_lock);
5641
5642         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5643
5644         die "VM $vmid already running\n" if $running;
5645
5646         if (my $storagemap = $migrate_opts->{storagemap}) {
5647             my $replicated = $migrate_opts->{replicated_volumes};
5648             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5649             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5650
5651             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5652                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5653             }
5654         }
5655
5656         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5657     });
5658 }
5659
5660
5661 # params:
5662 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5663 #   skiplock => 0/1, skip checking for config lock
5664 #   skiptemplate => 0/1, skip checking whether VM is template
5665 #   forcemachine => to force QEMU machine (rollback/migration)
5666 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5667 #   timeout => in seconds
5668 #   paused => start VM in paused state (backup)
5669 #   resume => resume from hibernation
5670 #   pbs-backing => {
5671 #      sata0 => {
5672 #         repository
5673 #         snapshot
5674 #         keyfile
5675 #         archive
5676 #      },
5677 #      virtio2 => ...
5678 #   }
5679 # migrate_opts:
5680 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5681 #   migratedfrom => source node
5682 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5683 #   network => CIDR of migration network
5684 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5685 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5686 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5687 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5688 #       contained in config
5689 sub vm_start_nolock {
5690     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5691
5692     my $statefile = $params->{statefile};
5693     my $resume = $params->{resume};
5694
5695     my $migratedfrom = $migrate_opts->{migratedfrom};
5696     my $migration_type = $migrate_opts->{type};
5697
5698     my $res = {};
5699
5700     # clean up leftover reboot request files
5701     eval { clear_reboot_request($vmid); };
5702     warn $@ if $@;
5703
5704     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5705         vmconfig_apply_pending($vmid, $conf, $storecfg);
5706         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5707     }
5708
5709     # don't regenerate the ISO if the VM is started as part of a live migration
5710     # this way we can reuse the old ISO with the correct config
5711     if (!$migratedfrom) {
5712         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5713             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5714             # $conf->{cloudinit}, so we could just not do this?
5715             # But we do it above, so for now let's be consistent.
5716             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5717         }
5718     }
5719
5720     # override offline migrated volumes, conf is out of date still
5721     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5722         for my $key (sort keys $offline_volumes->%*) {
5723             my $parsed = parse_drive($key, $conf->{$key});
5724             $parsed->{file} = $offline_volumes->{$key};
5725             $conf->{$key} = print_drive($parsed);
5726         }
5727     }
5728
5729     my $defaults = load_defaults();
5730
5731     # set environment variable useful inside network script
5732     # for remote migration the config is available on the target node!
5733     if (!$migrate_opts->{remote_node}) {
5734         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5735     }
5736
5737     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5738
5739     my $forcemachine = $params->{forcemachine};
5740     my $forcecpu = $params->{forcecpu};
5741     if ($resume) {
5742         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5743         $forcemachine = $conf->{runningmachine};
5744         $forcecpu = $conf->{runningcpu};
5745         print "Resuming suspended VM\n";
5746     }
5747
5748     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5749         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5750
5751     my $migration_ip;
5752     my $get_migration_ip = sub {
5753         my ($nodename) = @_;
5754
5755         return $migration_ip if defined($migration_ip);
5756
5757         my $cidr = $migrate_opts->{network};
5758
5759         if (!defined($cidr)) {
5760             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5761             $cidr = $dc_conf->{migration}->{network};
5762         }
5763
5764         if (defined($cidr)) {
5765             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5766
5767             die "could not get IP: no address configured on local " .
5768                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5769
5770             die "could not get IP: multiple addresses configured on local " .
5771                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5772
5773             $migration_ip = @$ips[0];
5774         }
5775
5776         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5777             if !defined($migration_ip);
5778
5779         return $migration_ip;
5780     };
5781
5782     if ($statefile) {
5783         if ($statefile eq 'tcp') {
5784             my $migrate = $res->{migrate} = { proto => 'tcp' };
5785             $migrate->{addr} = "localhost";
5786             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5787             my $nodename = nodename();
5788
5789             if (!defined($migration_type)) {
5790                 if (defined($datacenterconf->{migration}->{type})) {
5791                     $migration_type = $datacenterconf->{migration}->{type};
5792                 } else {
5793                     $migration_type = 'secure';
5794                 }
5795             }
5796
5797             if ($migration_type eq 'insecure') {
5798                 $migrate->{addr} = $get_migration_ip->($nodename);
5799                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5800             }
5801
5802             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5803             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5804             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5805             push @$cmd, '-incoming', $migrate->{uri};
5806             push @$cmd, '-S';
5807
5808         } elsif ($statefile eq 'unix') {
5809             # should be default for secure migrations as a ssh TCP forward
5810             # tunnel is not deterministic reliable ready and fails regurarly
5811             # to set up in time, so use UNIX socket forwards
5812             my $migrate = $res->{migrate} = { proto => 'unix' };
5813             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5814             unlink $migrate->{addr};
5815
5816             $migrate->{uri} = "unix:$migrate->{addr}";
5817             push @$cmd, '-incoming', $migrate->{uri};
5818             push @$cmd, '-S';
5819
5820         } elsif (-e $statefile) {
5821             push @$cmd, '-loadstate', $statefile;
5822         } else {
5823             my $statepath = PVE::Storage::path($storecfg, $statefile);
5824             push @$vollist, $statefile;
5825             push @$cmd, '-loadstate', $statepath;
5826         }
5827     } elsif ($params->{paused}) {
5828         push @$cmd, '-S';
5829     }
5830
5831     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5832
5833     my $pci_reserve_list = [];
5834     for my $device (values $pci_devices->%*) {
5835         next if $device->{mdev}; # we don't reserve for mdev devices
5836         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5837     }
5838
5839     # reserve all PCI IDs before actually doing anything with them
5840     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5841
5842     eval {
5843         my $uuid;
5844         for my $id (sort keys %$pci_devices) {
5845             my $d = $pci_devices->{$id};
5846             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5847
5848             my $chosen_mdev;
5849             for my $dev ($d->{ids}->@*) {
5850                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5851                 if ($d->{mdev}) {
5852                     warn $@ if $@;
5853                     $chosen_mdev = $info;
5854                     last if $chosen_mdev; # if successful, we're done
5855                 } else {
5856                     die $@ if $@;
5857                 }
5858             }
5859
5860             next if !$d->{mdev};
5861             die "could not create mediated device\n" if !defined($chosen_mdev);
5862
5863             # nvidia grid needs the uuid of the mdev as qemu parameter
5864             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5865                 if (defined($conf->{smbios1})) {
5866                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5867                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5868                 }
5869                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5870             }
5871         }
5872         push @$cmd, '-uuid', $uuid if defined($uuid);
5873     };
5874     if (my $err = $@) {
5875         eval { cleanup_pci_devices($vmid, $conf) };
5876         warn $@ if $@;
5877         die $err;
5878     }
5879
5880     PVE::Storage::activate_volumes($storecfg, $vollist);
5881
5882
5883     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5884     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5885     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5886     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5887     # timeout should be more than enough here...
5888     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5889
5890     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5891
5892     my %run_params = (
5893         timeout => $statefile ? undef : $start_timeout,
5894         umask => 0077,
5895         noerr => 1,
5896     );
5897
5898     # when migrating, prefix QEMU output so other side can pick up any
5899     # errors that might occur and show the user
5900     if ($migratedfrom) {
5901         $run_params{quiet} = 1;
5902         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5903     }
5904
5905     my %systemd_properties = (
5906         Slice => 'qemu.slice',
5907         KillMode => 'process',
5908         SendSIGKILL => 0,
5909         TimeoutStopUSec => ULONG_MAX, # infinity
5910     );
5911
5912     if (PVE::CGroup::cgroup_mode() == 2) {
5913         $systemd_properties{CPUWeight} = $cpuunits;
5914     } else {
5915         $systemd_properties{CPUShares} = $cpuunits;
5916     }
5917
5918     if (my $cpulimit = $conf->{cpulimit}) {
5919         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5920     }
5921     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5922
5923     my $run_qemu = sub {
5924         PVE::Tools::run_fork sub {
5925             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5926
5927             my $tpmpid;
5928             if (my $tpm = $conf->{tpmstate0}) {
5929                 # start the TPM emulator so QEMU can connect on start
5930                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5931             }
5932
5933             my $exitcode = run_command($cmd, %run_params);
5934             if ($exitcode) {
5935                 if ($tpmpid) {
5936                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5937                     kill 'TERM', $tpmpid;
5938                 }
5939                 die "QEMU exited with code $exitcode\n";
5940             }
5941         };
5942     };
5943
5944     if ($conf->{hugepages}) {
5945
5946         my $code = sub {
5947             my $hotplug_features =
5948                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5949             my $hugepages_topology =
5950                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5951
5952             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5953
5954             PVE::QemuServer::Memory::hugepages_mount();
5955             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5956
5957             eval { $run_qemu->() };
5958             if (my $err = $@) {
5959                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5960                     if !$conf->{keephugepages};
5961                 die $err;
5962             }
5963
5964             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5965                 if !$conf->{keephugepages};
5966         };
5967         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5968
5969     } else {
5970         eval { $run_qemu->() };
5971     }
5972
5973     if (my $err = $@) {
5974         # deactivate volumes if start fails
5975         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5976         warn $@ if $@;
5977         eval { cleanup_pci_devices($vmid, $conf) };
5978         warn $@ if $@;
5979
5980         die "start failed: $err";
5981     }
5982
5983     # re-reserve all PCI IDs now that we can know the actual VM PID
5984     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5985     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5986     warn $@ if $@;
5987
5988     if (defined($res->{migrate})) {
5989         print "migration listens on $res->{migrate}->{uri}\n";
5990     } elsif ($statefile) {
5991         eval { mon_cmd($vmid, "cont"); };
5992         warn $@ if $@;
5993     }
5994
5995     #start nbd server for storage migration
5996     if (my $nbd = $migrate_opts->{nbd}) {
5997         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5998
5999         my $migrate_storage_uri;
6000         # nbd_protocol_version > 0 for unix socket support
6001         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
6002             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
6003             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
6004             $migrate_storage_uri = "nbd:unix:$socket_path";
6005             $res->{migrate}->{unix_sockets} = [$socket_path];
6006         } else {
6007             my $nodename = nodename();
6008             my $localip = $get_migration_ip->($nodename);
6009             my $pfamily = PVE::Tools::get_host_address_family($nodename);
6010             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
6011
6012             mon_cmd($vmid, "nbd-server-start", addr => {
6013                 type => 'inet',
6014                 data => {
6015                     host => "${localip}",
6016                     port => "${storage_migrate_port}",
6017                 },
6018             });
6019             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
6020             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
6021         }
6022
6023         my $block_info = mon_cmd($vmid, "query-block");
6024         $block_info = { map { $_->{device} => $_ } $block_info->@* };
6025
6026         foreach my $opt (sort keys %$nbd) {
6027             my $drivestr = $nbd->{$opt}->{drivestr};
6028             my $volid = $nbd->{$opt}->{volid};
6029
6030             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
6031
6032             mon_cmd(
6033                 $vmid,
6034                 "block-export-add",
6035                 id => "drive-$opt",
6036                 'node-name' => $block_node,
6037                 writable => JSON::true,
6038                 type => "nbd",
6039                 name => "drive-$opt", # NBD export name
6040             );
6041
6042             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6043             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6044             print "re-using replicated volume: $opt - $volid\n"
6045                 if $nbd->{$opt}->{replicated};
6046
6047             $res->{drives}->{$opt} = $nbd->{$opt};
6048             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6049         }
6050     }
6051
6052     if ($migratedfrom) {
6053         eval {
6054             set_migration_caps($vmid);
6055         };
6056         warn $@ if $@;
6057
6058         if ($spice_port) {
6059             print "spice listens on port $spice_port\n";
6060             $res->{spice_port} = $spice_port;
6061             if ($migrate_opts->{spice_ticket}) {
6062                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6063                     $migrate_opts->{spice_ticket});
6064                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6065             }
6066         }
6067
6068     } else {
6069         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6070             if !$statefile && $conf->{balloon};
6071
6072         foreach my $opt (keys %$conf) {
6073             next if $opt !~  m/^net\d+$/;
6074             my $nicconf = parse_net($conf->{$opt});
6075             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6076         }
6077         add_nets_bridge_fdb($conf, $vmid);
6078     }
6079
6080    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6081         eval {
6082             mon_cmd(
6083                 $vmid,
6084                 'qom-set',
6085                 path => "machine/peripheral/balloon0",
6086                 property => "guest-stats-polling-interval",
6087                 value => 2
6088             );
6089         };
6090         log_warn("could not set polling interval for ballooning - $@") if $@;
6091     }
6092
6093     if ($resume) {
6094         print "Resumed VM, removing state\n";
6095         if (my $vmstate = $conf->{vmstate}) {
6096             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6097             PVE::Storage::vdisk_free($storecfg, $vmstate);
6098         }
6099         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6100         PVE::QemuConfig->write_config($vmid, $conf);
6101     }
6102
6103     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6104
6105     return $res;
6106 }
6107
6108 sub vm_commandline {
6109     my ($storecfg, $vmid, $snapname) = @_;
6110
6111     my $conf = PVE::QemuConfig->load_config($vmid);
6112
6113     my ($forcemachine, $forcecpu);
6114     if ($snapname) {
6115         my $snapshot = $conf->{snapshots}->{$snapname};
6116         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6117
6118         # check for machine or CPU overrides in snapshot
6119         $forcemachine = $snapshot->{runningmachine};
6120         $forcecpu = $snapshot->{runningcpu};
6121
6122         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6123
6124         $conf = $snapshot;
6125     }
6126
6127     my $defaults = load_defaults();
6128
6129     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6130
6131     return PVE::Tools::cmd2string($cmd);
6132 }
6133
6134 sub vm_reset {
6135     my ($vmid, $skiplock) = @_;
6136
6137     PVE::QemuConfig->lock_config($vmid, sub {
6138
6139         my $conf = PVE::QemuConfig->load_config($vmid);
6140
6141         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6142
6143         mon_cmd($vmid, "system_reset");
6144     });
6145 }
6146
6147 sub get_vm_volumes {
6148     my ($conf) = @_;
6149
6150     my $vollist = [];
6151     foreach_volid($conf, 1, sub {
6152         my ($volid, $attr) = @_;
6153
6154         return if $volid =~ m|^/|;
6155
6156         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6157         return if !$sid;
6158
6159         push @$vollist, $volid;
6160     });
6161
6162     return $vollist;
6163 }
6164
6165 sub cleanup_pci_devices {
6166     my ($vmid, $conf) = @_;
6167
6168     foreach my $key (keys %$conf) {
6169         next if $key !~ m/^hostpci(\d+)$/;
6170         my $hostpciindex = $1;
6171         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6172         my $d = parse_hostpci($conf->{$key});
6173         if ($d->{mdev}) {
6174             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6175             # don't want to break ABI just for this two liner
6176             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6177
6178             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6179             # out when we do it first. so wait for 10 seconds and then try it
6180             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/) {
6181                 sleep 10;
6182             }
6183
6184             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6185         }
6186     }
6187     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6188 }
6189
6190 sub vm_stop_cleanup {
6191     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6192
6193     eval {
6194
6195         if (!$keepActive) {
6196             my $vollist = get_vm_volumes($conf);
6197             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6198
6199             if (my $tpmdrive = $conf->{tpmstate0}) {
6200                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6201                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6202                 if ($storeid) {
6203                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6204                 }
6205             }
6206         }
6207
6208         foreach my $ext (qw(mon qmp pid vnc qga)) {
6209             unlink "/var/run/qemu-server/${vmid}.$ext";
6210         }
6211
6212         if ($conf->{ivshmem}) {
6213             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6214             # just delete it for now, VMs which have this already open do not
6215             # are affected, but new VMs will get a separated one. If this
6216             # becomes an issue we either add some sort of ref-counting or just
6217             # add a "don't delete on stop" flag to the ivshmem format.
6218             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6219         }
6220
6221         cleanup_pci_devices($vmid, $conf);
6222
6223         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6224     };
6225     warn $@ if $@; # avoid errors - just warn
6226 }
6227
6228 # call only in locked context
6229 sub _do_vm_stop {
6230     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6231
6232     my $pid = check_running($vmid, $nocheck);
6233     return if !$pid;
6234
6235     my $conf;
6236     if (!$nocheck) {
6237         $conf = PVE::QemuConfig->load_config($vmid);
6238         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6239         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6240             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6241             $timeout = $opts->{down} if $opts->{down};
6242         }
6243         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6244     }
6245
6246     eval {
6247         if ($shutdown) {
6248             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6249                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6250             } else {
6251                 mon_cmd($vmid, "system_powerdown");
6252             }
6253         } else {
6254             mon_cmd($vmid, "quit");
6255         }
6256     };
6257     my $err = $@;
6258
6259     if (!$err) {
6260         $timeout = 60 if !defined($timeout);
6261
6262         my $count = 0;
6263         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6264             $count++;
6265             sleep 1;
6266         }
6267
6268         if ($count >= $timeout) {
6269             if ($force) {
6270                 warn "VM still running - terminating now with SIGTERM\n";
6271                 kill 15, $pid;
6272             } else {
6273                 die "VM quit/powerdown failed - got timeout\n";
6274             }
6275         } else {
6276             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6277             return;
6278         }
6279     } else {
6280        if (!check_running($vmid, $nocheck)) {
6281             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6282             return;
6283        }
6284        if ($force) {
6285             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6286             kill 15, $pid;
6287         } else {
6288             die "VM quit/powerdown failed\n";
6289         }
6290     }
6291
6292     # wait again
6293     $timeout = 10;
6294
6295     my $count = 0;
6296     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6297         $count++;
6298         sleep 1;
6299     }
6300
6301     if ($count >= $timeout) {
6302         warn "VM still running - terminating now with SIGKILL\n";
6303         kill 9, $pid;
6304         sleep 1;
6305     }
6306
6307     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6308 }
6309
6310 # Note: use $nocheck to skip tests if VM configuration file exists.
6311 # We need that when migration VMs to other nodes (files already moved)
6312 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6313 sub vm_stop {
6314     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6315
6316     $force = 1 if !defined($force) && !$shutdown;
6317
6318     if ($migratedfrom){
6319         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6320         kill 15, $pid if $pid;
6321         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6322         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6323         return;
6324     }
6325
6326     PVE::QemuConfig->lock_config($vmid, sub {
6327         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6328    });
6329 }
6330
6331 sub vm_reboot {
6332     my ($vmid, $timeout) = @_;
6333
6334     PVE::QemuConfig->lock_config($vmid, sub {
6335         eval {
6336
6337             # only reboot if running, as qmeventd starts it again on a stop event
6338             return if !check_running($vmid);
6339
6340             create_reboot_request($vmid);
6341
6342             my $storecfg = PVE::Storage::config();
6343             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6344
6345         };
6346         if (my $err = $@) {
6347             # avoid that the next normal shutdown will be confused for a reboot
6348             clear_reboot_request($vmid);
6349             die $err;
6350         }
6351    });
6352 }
6353
6354 # note: if using the statestorage parameter, the caller has to check privileges
6355 sub vm_suspend {
6356     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6357
6358     my $conf;
6359     my $path;
6360     my $storecfg;
6361     my $vmstate;
6362
6363     PVE::QemuConfig->lock_config($vmid, sub {
6364
6365         $conf = PVE::QemuConfig->load_config($vmid);
6366
6367         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6368         PVE::QemuConfig->check_lock($conf)
6369             if !($skiplock || $is_backing_up);
6370
6371         die "cannot suspend to disk during backup\n"
6372             if $is_backing_up && $includestate;
6373
6374         if ($includestate) {
6375             $conf->{lock} = 'suspending';
6376             my $date = strftime("%Y-%m-%d", localtime(time()));
6377             $storecfg = PVE::Storage::config();
6378             if (!$statestorage) {
6379                 $statestorage = find_vmstate_storage($conf, $storecfg);
6380                 # check permissions for the storage
6381                 my $rpcenv = PVE::RPCEnvironment::get();
6382                 if ($rpcenv->{type} ne 'cli') {
6383                     my $authuser = $rpcenv->get_user();
6384                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6385                 }
6386             }
6387
6388
6389             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6390                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6391             $path = PVE::Storage::path($storecfg, $vmstate);
6392             PVE::QemuConfig->write_config($vmid, $conf);
6393         } else {
6394             mon_cmd($vmid, "stop");
6395         }
6396     });
6397
6398     if ($includestate) {
6399         # save vm state
6400         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6401
6402         eval {
6403             set_migration_caps($vmid, 1);
6404             mon_cmd($vmid, "savevm-start", statefile => $path);
6405             for(;;) {
6406                 my $state = mon_cmd($vmid, "query-savevm");
6407                 if (!$state->{status}) {
6408                     die "savevm not active\n";
6409                 } elsif ($state->{status} eq 'active') {
6410                     sleep(1);
6411                     next;
6412                 } elsif ($state->{status} eq 'completed') {
6413                     print "State saved, quitting\n";
6414                     last;
6415                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6416                     die "query-savevm failed with error '$state->{error}'\n"
6417                 } else {
6418                     die "query-savevm returned status '$state->{status}'\n";
6419                 }
6420             }
6421         };
6422         my $err = $@;
6423
6424         PVE::QemuConfig->lock_config($vmid, sub {
6425             $conf = PVE::QemuConfig->load_config($vmid);
6426             if ($err) {
6427                 # cleanup, but leave suspending lock, to indicate something went wrong
6428                 eval {
6429                     mon_cmd($vmid, "savevm-end");
6430                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6431                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6432                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6433                     PVE::QemuConfig->write_config($vmid, $conf);
6434                 };
6435                 warn $@ if $@;
6436                 die $err;
6437             }
6438
6439             die "lock changed unexpectedly\n"
6440                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6441
6442             mon_cmd($vmid, "quit");
6443             $conf->{lock} = 'suspended';
6444             PVE::QemuConfig->write_config($vmid, $conf);
6445         });
6446     }
6447 }
6448
6449 # $nocheck is set when called as part of a migration - in this context the
6450 # location of the config file (source or target node) is not deterministic,
6451 # since migration cannot wait for pmxcfs to process the rename
6452 sub vm_resume {
6453     my ($vmid, $skiplock, $nocheck) = @_;
6454
6455     PVE::QemuConfig->lock_config($vmid, sub {
6456         my $res = mon_cmd($vmid, 'query-status');
6457         my $resume_cmd = 'cont';
6458         my $reset = 0;
6459         my $conf;
6460         if ($nocheck) {
6461             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6462             if ($@) {
6463                 my $vmlist = PVE::Cluster::get_vmlist();
6464                 if (exists($vmlist->{ids}->{$vmid})) {
6465                     my $node = $vmlist->{ids}->{$vmid}->{node};
6466                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6467                 }
6468                 if (!$conf) {
6469                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6470                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6471                 }
6472             }
6473         } else {
6474             $conf = PVE::QemuConfig->load_config($vmid);
6475         }
6476
6477         if ($res->{status}) {
6478             return if $res->{status} eq 'running'; # job done, go home
6479             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6480             $reset = 1 if $res->{status} eq 'shutdown';
6481         }
6482
6483         if (!$nocheck) {
6484             PVE::QemuConfig->check_lock($conf)
6485                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6486         }
6487
6488         if ($reset) {
6489             # required if a VM shuts down during a backup and we get a resume
6490             # request before the backup finishes for example
6491             mon_cmd($vmid, "system_reset");
6492         }
6493
6494         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6495
6496         mon_cmd($vmid, $resume_cmd);
6497     });
6498 }
6499
6500 sub vm_sendkey {
6501     my ($vmid, $skiplock, $key) = @_;
6502
6503     PVE::QemuConfig->lock_config($vmid, sub {
6504
6505         my $conf = PVE::QemuConfig->load_config($vmid);
6506
6507         # there is no qmp command, so we use the human monitor command
6508         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6509         die $res if $res ne '';
6510     });
6511 }
6512
6513 sub check_bridge_access {
6514     my ($rpcenv, $authuser, $conf) = @_;
6515
6516     return 1 if $authuser eq 'root@pam';
6517
6518     for my $opt (sort keys $conf->%*) {
6519         next if $opt !~ m/^net\d+$/;
6520         my $net = parse_net($conf->{$opt});
6521         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6522         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6523     }
6524     return 1;
6525 };
6526
6527 sub check_mapping_access {
6528     my ($rpcenv, $user, $conf) = @_;
6529
6530     for my $opt (keys $conf->%*) {
6531         if ($opt =~ m/^usb\d+$/) {
6532             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6533             if (my $host = $device->{host}) {
6534                 die "only root can set '$opt' config for real devices\n"
6535                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6536             } elsif ($device->{mapping}) {
6537                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6538             } else {
6539                 die "either 'host' or 'mapping' must be set.\n";
6540             }
6541         } elsif ($opt =~ m/^hostpci\d+$/) {
6542             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6543             if ($device->{host}) {
6544                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6545             } elsif ($device->{mapping}) {
6546                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6547             } else {
6548                 die "either 'host' or 'mapping' must be set.\n";
6549             }
6550        }
6551    }
6552 };
6553
6554 sub check_restore_permissions {
6555     my ($rpcenv, $user, $conf) = @_;
6556
6557     check_bridge_access($rpcenv, $user, $conf);
6558     check_mapping_access($rpcenv, $user, $conf);
6559 }
6560 # vzdump restore implementaion
6561
6562 sub tar_archive_read_firstfile {
6563     my $archive = shift;
6564
6565     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6566
6567     # try to detect archive type first
6568     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6569         die "unable to open file '$archive'\n";
6570     my $firstfile = <$fh>;
6571     kill 15, $pid;
6572     close $fh;
6573
6574     die "ERROR: archive contaions no data\n" if !$firstfile;
6575     chomp $firstfile;
6576
6577     return $firstfile;
6578 }
6579
6580 sub tar_restore_cleanup {
6581     my ($storecfg, $statfile) = @_;
6582
6583     print STDERR "starting cleanup\n";
6584
6585     if (my $fd = IO::File->new($statfile, "r")) {
6586         while (defined(my $line = <$fd>)) {
6587             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6588                 my $volid = $2;
6589                 eval {
6590                     if ($volid =~ m|^/|) {
6591                         unlink $volid || die 'unlink failed\n';
6592                     } else {
6593                         PVE::Storage::vdisk_free($storecfg, $volid);
6594                     }
6595                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6596                 };
6597                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6598             } else {
6599                 print STDERR "unable to parse line in statfile - $line";
6600             }
6601         }
6602         $fd->close();
6603     }
6604 }
6605
6606 sub restore_file_archive {
6607     my ($archive, $vmid, $user, $opts) = @_;
6608
6609     return restore_vma_archive($archive, $vmid, $user, $opts)
6610         if $archive eq '-';
6611
6612     my $info = PVE::Storage::archive_info($archive);
6613     my $format = $opts->{format} // $info->{format};
6614     my $comp = $info->{compression};
6615
6616     # try to detect archive format
6617     if ($format eq 'tar') {
6618         return restore_tar_archive($archive, $vmid, $user, $opts);
6619     } else {
6620         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6621     }
6622 }
6623
6624 # hepler to remove disks that will not be used after restore
6625 my $restore_cleanup_oldconf = sub {
6626     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6627
6628     my $kept_disks = {};
6629
6630     PVE::QemuConfig->foreach_volume($oldconf, sub {
6631         my ($ds, $drive) = @_;
6632
6633         return if drive_is_cdrom($drive, 1);
6634
6635         my $volid = $drive->{file};
6636         return if !$volid || $volid =~ m|^/|;
6637
6638         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6639         return if !$path || !$owner || ($owner != $vmid);
6640
6641         # Note: only delete disk we want to restore
6642         # other volumes will become unused
6643         if ($virtdev_hash->{$ds}) {
6644             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6645             if (my $err = $@) {
6646                 warn $err;
6647             }
6648         } else {
6649             $kept_disks->{$volid} = 1;
6650         }
6651     });
6652
6653     # after the restore we have no snapshots anymore
6654     for my $snapname (keys $oldconf->{snapshots}->%*) {
6655         my $snap = $oldconf->{snapshots}->{$snapname};
6656         if ($snap->{vmstate}) {
6657             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6658             if (my $err = $@) {
6659                 warn $err;
6660             }
6661         }
6662
6663         for my $volid (keys $kept_disks->%*) {
6664             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6665             warn $@ if $@;
6666         }
6667     }
6668 };
6669
6670 # Helper to parse vzdump backup device hints
6671 #
6672 # $rpcenv: Environment, used to ckeck storage permissions
6673 # $user: User ID, to check storage permissions
6674 # $storecfg: Storage configuration
6675 # $fh: the file handle for reading the configuration
6676 # $devinfo: should contain device sizes for all backu-up'ed devices
6677 # $options: backup options (pool, default storage)
6678 #
6679 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6680 my $parse_backup_hints = sub {
6681     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6682
6683     my $check_storage = sub { # assert if an image can be allocate
6684         my ($storeid, $scfg) = @_;
6685         die "Content type 'images' is not available on storage '$storeid'\n"
6686             if !$scfg->{content}->{images};
6687         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6688             if $user ne 'root@pam';
6689     };
6690
6691     my $virtdev_hash = {};
6692     while (defined(my $line = <$fh>)) {
6693         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6694             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6695             die "archive does not contain data for drive '$virtdev'\n"
6696                 if !$devinfo->{$devname};
6697
6698             if (defined($options->{storage})) {
6699                 $storeid = $options->{storage} || 'local';
6700             } elsif (!$storeid) {
6701                 $storeid = 'local';
6702             }
6703             $format = 'raw' if !$format;
6704             $devinfo->{$devname}->{devname} = $devname;
6705             $devinfo->{$devname}->{virtdev} = $virtdev;
6706             $devinfo->{$devname}->{format} = $format;
6707             $devinfo->{$devname}->{storeid} = $storeid;
6708
6709             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6710             $check_storage->($storeid, $scfg); # permission and content type check
6711
6712             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6713         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6714             my $virtdev = $1;
6715             my $drive = parse_drive($virtdev, $2);
6716
6717             if (drive_is_cloudinit($drive)) {
6718                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6719                 $storeid = $options->{storage} if defined ($options->{storage});
6720                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6721                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6722
6723                 $check_storage->($storeid, $scfg); # permission and content type check
6724
6725                 $virtdev_hash->{$virtdev} = {
6726                     format => $format,
6727                     storeid => $storeid,
6728                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6729                     is_cloudinit => 1,
6730                 };
6731             }
6732         }
6733     }
6734
6735     return $virtdev_hash;
6736 };
6737
6738 # Helper to allocate and activate all volumes required for a restore
6739 #
6740 # $storecfg: Storage configuration
6741 # $virtdev_hash: as returned by parse_backup_hints()
6742 #
6743 # Returns: { $virtdev => $volid }
6744 my $restore_allocate_devices = sub {
6745     my ($storecfg, $virtdev_hash, $vmid) = @_;
6746
6747     my $map = {};
6748     foreach my $virtdev (sort keys %$virtdev_hash) {
6749         my $d = $virtdev_hash->{$virtdev};
6750         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6751         my $storeid = $d->{storeid};
6752         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6753
6754         # test if requested format is supported
6755         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6756         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6757         $d->{format} = $defFormat if !$supported;
6758
6759         my $name;
6760         if ($d->{is_cloudinit}) {
6761             $name = "vm-$vmid-cloudinit";
6762             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6763             if ($scfg->{path}) {
6764                 $name .= ".$d->{format}";
6765             }
6766         }
6767
6768         my $volid = PVE::Storage::vdisk_alloc(
6769             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6770
6771         print STDERR "new volume ID is '$volid'\n";
6772         $d->{volid} = $volid;
6773
6774         PVE::Storage::activate_volumes($storecfg, [$volid]);
6775
6776         $map->{$virtdev} = $volid;
6777     }
6778
6779     return $map;
6780 };
6781
6782 sub restore_update_config_line {
6783     my ($cookie, $map, $line, $unique) = @_;
6784
6785     return '' if $line =~ m/^\#qmdump\#/;
6786     return '' if $line =~ m/^\#vzdump\#/;
6787     return '' if $line =~ m/^lock:/;
6788     return '' if $line =~ m/^unused\d+:/;
6789     return '' if $line =~ m/^parent:/;
6790
6791     my $res = '';
6792
6793     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6794     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6795         # try to convert old 1.X settings
6796         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6797         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6798             my ($model, $macaddr) = split(/\=/, $devconfig);
6799             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6800             my $net = {
6801                 model => $model,
6802                 bridge => "vmbr$ind",
6803                 macaddr => $macaddr,
6804             };
6805             my $netstr = print_net($net);
6806
6807             $res .= "net$cookie->{netcount}: $netstr\n";
6808             $cookie->{netcount}++;
6809         }
6810     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6811         my ($id, $netstr) = ($1, $2);
6812         my $net = parse_net($netstr);
6813         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6814         $netstr = print_net($net);
6815         $res .= "$id: $netstr\n";
6816     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6817         my $virtdev = $1;
6818         my $value = $3;
6819         my $di = parse_drive($virtdev, $value);
6820         if (defined($di->{backup}) && !$di->{backup}) {
6821             $res .= "#$line";
6822         } elsif ($map->{$virtdev}) {
6823             delete $di->{format}; # format can change on restore
6824             $di->{file} = $map->{$virtdev};
6825             $value = print_drive($di);
6826             $res .= "$virtdev: $value\n";
6827         } else {
6828             $res .= $line;
6829         }
6830     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6831         my $vmgenid = $1;
6832         if ($vmgenid ne '0') {
6833             # always generate a new vmgenid if there was a valid one setup
6834             $vmgenid = generate_uuid();
6835         }
6836         $res .= "vmgenid: $vmgenid\n";
6837     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6838         my ($uuid, $uuid_str);
6839         UUID::generate($uuid);
6840         UUID::unparse($uuid, $uuid_str);
6841         my $smbios1 = parse_smbios1($2);
6842         $smbios1->{uuid} = $uuid_str;
6843         $res .= $1.print_smbios1($smbios1)."\n";
6844     } else {
6845         $res .= $line;
6846     }
6847
6848     return $res;
6849 }
6850
6851 my $restore_deactivate_volumes = sub {
6852     my ($storecfg, $virtdev_hash) = @_;
6853
6854     my $vollist = [];
6855     for my $dev (values $virtdev_hash->%*) {
6856         push $vollist->@*, $dev->{volid} if $dev->{volid};
6857     }
6858
6859     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6860     print STDERR $@ if $@;
6861 };
6862
6863 my $restore_destroy_volumes = sub {
6864     my ($storecfg, $virtdev_hash) = @_;
6865
6866     for my $dev (values $virtdev_hash->%*) {
6867         my $volid = $dev->{volid} or next;
6868         eval {
6869             PVE::Storage::vdisk_free($storecfg, $volid);
6870             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6871         };
6872         print STDERR "unable to cleanup '$volid' - $@" if $@;
6873     }
6874 };
6875
6876 sub restore_merge_config {
6877     my ($filename, $backup_conf_raw, $override_conf) = @_;
6878
6879     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6880     for my $key (keys $override_conf->%*) {
6881         $backup_conf->{$key} = $override_conf->{$key};
6882     }
6883
6884     return $backup_conf;
6885 }
6886
6887 sub scan_volids {
6888     my ($cfg, $vmid) = @_;
6889
6890     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6891
6892     my $volid_hash = {};
6893     foreach my $storeid (keys %$info) {
6894         foreach my $item (@{$info->{$storeid}}) {
6895             next if !($item->{volid} && $item->{size});
6896             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6897             $volid_hash->{$item->{volid}} = $item;
6898         }
6899     }
6900
6901     return $volid_hash;
6902 }
6903
6904 sub update_disk_config {
6905     my ($vmid, $conf, $volid_hash) = @_;
6906
6907     my $changes;
6908     my $prefix = "VM $vmid";
6909
6910     # used and unused disks
6911     my $referenced = {};
6912
6913     # Note: it is allowed to define multiple storages with same path (alias), so
6914     # we need to check both 'volid' and real 'path' (two different volid can point
6915     # to the same path).
6916
6917     my $referencedpath = {};
6918
6919     # update size info
6920     PVE::QemuConfig->foreach_volume($conf, sub {
6921         my ($opt, $drive) = @_;
6922
6923         my $volid = $drive->{file};
6924         return if !$volid;
6925         my $volume = $volid_hash->{$volid};
6926
6927         # mark volid as "in-use" for next step
6928         $referenced->{$volid} = 1;
6929         if ($volume && (my $path = $volume->{path})) {
6930             $referencedpath->{$path} = 1;
6931         }
6932
6933         return if drive_is_cdrom($drive);
6934         return if !$volume;
6935
6936         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6937         if (defined($updated)) {
6938             $changes = 1;
6939             $conf->{$opt} = print_drive($updated);
6940             print "$prefix ($opt): $msg\n";
6941         }
6942     });
6943
6944     # remove 'unusedX' entry if volume is used
6945     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6946         my ($opt, $drive) = @_;
6947
6948         my $volid = $drive->{file};
6949         return if !$volid;
6950
6951         my $path;
6952         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6953         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6954             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6955             $changes = 1;
6956             delete $conf->{$opt};
6957         }
6958
6959         $referenced->{$volid} = 1;
6960         $referencedpath->{$path} = 1 if $path;
6961     });
6962
6963     foreach my $volid (sort keys %$volid_hash) {
6964         next if $volid =~ m/vm-$vmid-state-/;
6965         next if $referenced->{$volid};
6966         my $path = $volid_hash->{$volid}->{path};
6967         next if !$path; # just to be sure
6968         next if $referencedpath->{$path};
6969         $changes = 1;
6970         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6971         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6972         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6973     }
6974
6975     return $changes;
6976 }
6977
6978 sub rescan {
6979     my ($vmid, $nolock, $dryrun) = @_;
6980
6981     my $cfg = PVE::Storage::config();
6982
6983     print "rescan volumes...\n";
6984     my $volid_hash = scan_volids($cfg, $vmid);
6985
6986     my $updatefn =  sub {
6987         my ($vmid) = @_;
6988
6989         my $conf = PVE::QemuConfig->load_config($vmid);
6990
6991         PVE::QemuConfig->check_lock($conf);
6992
6993         my $vm_volids = {};
6994         foreach my $volid (keys %$volid_hash) {
6995             my $info = $volid_hash->{$volid};
6996             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6997         }
6998
6999         my $changes = update_disk_config($vmid, $conf, $vm_volids);
7000
7001         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
7002     };
7003
7004     if (defined($vmid)) {
7005         if ($nolock) {
7006             &$updatefn($vmid);
7007         } else {
7008             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
7009         }
7010     } else {
7011         my $vmlist = config_list();
7012         foreach my $vmid (keys %$vmlist) {
7013             if ($nolock) {
7014                 &$updatefn($vmid);
7015             } else {
7016                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
7017             }
7018         }
7019     }
7020 }
7021
7022 sub restore_proxmox_backup_archive {
7023     my ($archive, $vmid, $user, $options) = @_;
7024
7025     my $storecfg = PVE::Storage::config();
7026
7027     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
7028     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7029
7030     my $fingerprint = $scfg->{fingerprint};
7031     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
7032
7033     my $repo = PVE::PBSClient::get_repository($scfg);
7034     my $namespace = $scfg->{namespace};
7035
7036     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7037     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7038     local $ENV{PBS_PASSWORD} = $password;
7039     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7040
7041     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7042         PVE::Storage::parse_volname($storecfg, $archive);
7043
7044     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7045
7046     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7047
7048     my $tmpdir = "/var/tmp/vzdumptmp$$";
7049     rmtree $tmpdir;
7050     mkpath $tmpdir;
7051
7052     my $conffile = PVE::QemuConfig->config_file($vmid);
7053      # disable interrupts (always do cleanups)
7054     local $SIG{INT} =
7055         local $SIG{TERM} =
7056         local $SIG{QUIT} =
7057         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7058
7059     # Note: $oldconf is undef if VM does not exists
7060     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7061     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7062     my $new_conf_raw = '';
7063
7064     my $rpcenv = PVE::RPCEnvironment::get();
7065     my $devinfo = {}; # info about drives included in backup
7066     my $virtdev_hash = {}; # info about allocated drives
7067
7068     eval {
7069         # enable interrupts
7070         local $SIG{INT} =
7071             local $SIG{TERM} =
7072             local $SIG{QUIT} =
7073             local $SIG{HUP} =
7074             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7075
7076         my $cfgfn = "$tmpdir/qemu-server.conf";
7077         my $firewall_config_fn = "$tmpdir/fw.conf";
7078         my $index_fn = "$tmpdir/index.json";
7079
7080         my $cmd = "restore";
7081
7082         my $param = [$pbs_backup_name, "index.json", $index_fn];
7083         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7084         my $index = PVE::Tools::file_get_contents($index_fn);
7085         $index = decode_json($index);
7086
7087         foreach my $info (@{$index->{files}}) {
7088             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7089                 my $devname = $1;
7090                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7091                     $devinfo->{$devname}->{size} = $1;
7092                 } else {
7093                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7094                 }
7095             }
7096         }
7097
7098         my $is_qemu_server_backup = scalar(
7099             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7100         );
7101         if (!$is_qemu_server_backup) {
7102             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7103         }
7104         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7105
7106         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7107         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7108
7109         if ($has_firewall_config) {
7110             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7111             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7112
7113             my $pve_firewall_dir = '/etc/pve/firewall';
7114             mkdir $pve_firewall_dir; # make sure the dir exists
7115             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7116         }
7117
7118         my $fh = IO::File->new($cfgfn, "r") ||
7119             die "unable to read qemu-server.conf - $!\n";
7120
7121         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7122
7123         # fixme: rate limit?
7124
7125         # create empty/temp config
7126         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7127
7128         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7129
7130         # allocate volumes
7131         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7132
7133         foreach my $virtdev (sort keys %$virtdev_hash) {
7134             my $d = $virtdev_hash->{$virtdev};
7135             next if $d->{is_cloudinit}; # no need to restore cloudinit
7136
7137             # this fails if storage is unavailable
7138             my $volid = $d->{volid};
7139             my $path = PVE::Storage::path($storecfg, $volid);
7140
7141             # for live-restore we only want to preload the efidisk and TPM state
7142             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7143
7144             my @ns_arg;
7145             if (defined(my $ns = $scfg->{namespace})) {
7146                 @ns_arg = ('--ns', $ns);
7147             }
7148
7149             my $pbs_restore_cmd = [
7150                 '/usr/bin/pbs-restore',
7151                 '--repository', $repo,
7152                 @ns_arg,
7153                 $pbs_backup_name,
7154                 "$d->{devname}.img.fidx",
7155                 $path,
7156                 '--verbose',
7157                 ];
7158
7159             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7160             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7161
7162             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7163                 push @$pbs_restore_cmd, '--skip-zero';
7164             }
7165
7166             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7167             print "restore proxmox backup image: $dbg_cmdstring\n";
7168             run_command($pbs_restore_cmd);
7169         }
7170
7171         $fh->seek(0, 0) || die "seek failed - $!\n";
7172
7173         my $cookie = { netcount => 0 };
7174         while (defined(my $line = <$fh>)) {
7175             $new_conf_raw .= restore_update_config_line(
7176                 $cookie,
7177                 $map,
7178                 $line,
7179                 $options->{unique},
7180             );
7181         }
7182
7183         $fh->close();
7184     };
7185     my $err = $@;
7186
7187     if ($err || !$options->{live}) {
7188         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7189     }
7190
7191     rmtree $tmpdir;
7192
7193     if ($err) {
7194         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7195         die $err;
7196     }
7197
7198     if ($options->{live}) {
7199         # keep lock during live-restore
7200         $new_conf_raw .= "\nlock: create";
7201     }
7202
7203     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7204     check_restore_permissions($rpcenv, $user, $new_conf);
7205     PVE::QemuConfig->write_config($vmid, $new_conf);
7206
7207     eval { rescan($vmid, 1); };
7208     warn $@ if $@;
7209
7210     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7211
7212     if ($options->{live}) {
7213         # enable interrupts
7214         local $SIG{INT} =
7215             local $SIG{TERM} =
7216             local $SIG{QUIT} =
7217             local $SIG{HUP} =
7218             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7219
7220         my $conf = PVE::QemuConfig->load_config($vmid);
7221         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7222
7223         # these special drives are already restored before start
7224         delete $devinfo->{'drive-efidisk0'};
7225         delete $devinfo->{'drive-tpmstate0-backup'};
7226
7227         my $pbs_opts = {
7228             repo => $repo,
7229             keyfile => $keyfile,
7230             snapshot => $pbs_backup_name,
7231             namespace => $namespace,
7232         };
7233         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7234
7235         PVE::QemuConfig->remove_lock($vmid, "create");
7236     }
7237 }
7238
7239 sub pbs_live_restore {
7240     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7241
7242     print "starting VM for live-restore\n";
7243     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7244
7245     my $pbs_backing = {};
7246     for my $ds (keys %$restored_disks) {
7247         $ds =~ m/^drive-(.*)$/;
7248         my $confname = $1;
7249         $pbs_backing->{$confname} = {
7250             repository => $opts->{repo},
7251             snapshot => $opts->{snapshot},
7252             archive => "$ds.img.fidx",
7253         };
7254         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7255         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7256
7257         my $drive = parse_drive($confname, $conf->{$confname});
7258         print "restoring '$ds' to '$drive->{file}'\n";
7259     }
7260
7261     my $drives_streamed = 0;
7262     eval {
7263         # make sure HA doesn't interrupt our restore by stopping the VM
7264         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7265             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7266         }
7267
7268         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7269         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7270         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7271
7272         my $qmeventd_fd = register_qmeventd_handle($vmid);
7273
7274         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7275         # this will effectively collapse the backing image chain consisting of
7276         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7277         # removes itself once all backing images vanish with 'auto-remove=on')
7278         my $jobs = {};
7279         for my $ds (sort keys %$restored_disks) {
7280             my $job_id = "restore-$ds";
7281             mon_cmd($vmid, 'block-stream',
7282                 'job-id' => $job_id,
7283                 device => "$ds",
7284             );
7285             $jobs->{$job_id} = {};
7286         }
7287
7288         mon_cmd($vmid, 'cont');
7289         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7290
7291         print "restore-drive jobs finished successfully, removing all tracking block devices"
7292             ." to disconnect from Proxmox Backup Server\n";
7293
7294         for my $ds (sort keys %$restored_disks) {
7295             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7296         }
7297
7298         close($qmeventd_fd);
7299     };
7300
7301     my $err = $@;
7302
7303     if ($err) {
7304         warn "An error occurred during live-restore: $err\n";
7305         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7306         die "live-restore failed\n";
7307     }
7308 }
7309
7310 sub restore_vma_archive {
7311     my ($archive, $vmid, $user, $opts, $comp) = @_;
7312
7313     my $readfrom = $archive;
7314
7315     my $cfg = PVE::Storage::config();
7316     my $commands = [];
7317     my $bwlimit = $opts->{bwlimit};
7318
7319     my $dbg_cmdstring = '';
7320     my $add_pipe = sub {
7321         my ($cmd) = @_;
7322         push @$commands, $cmd;
7323         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7324         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7325         $readfrom = '-';
7326     };
7327
7328     my $input = undef;
7329     if ($archive eq '-') {
7330         $input = '<&STDIN';
7331     } else {
7332         # If we use a backup from a PVE defined storage we also consider that
7333         # storage's rate limit:
7334         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7335         if (defined($volid)) {
7336             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7337             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7338             if ($readlimit) {
7339                 print STDERR "applying read rate limit: $readlimit\n";
7340                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7341                 $add_pipe->($cstream);
7342             }
7343         }
7344     }
7345
7346     if ($comp) {
7347         my $info = PVE::Storage::decompressor_info('vma', $comp);
7348         my $cmd = $info->{decompressor};
7349         push @$cmd, $readfrom;
7350         $add_pipe->($cmd);
7351     }
7352
7353     my $tmpdir = "/var/tmp/vzdumptmp$$";
7354     rmtree $tmpdir;
7355
7356     # disable interrupts (always do cleanups)
7357     local $SIG{INT} =
7358         local $SIG{TERM} =
7359         local $SIG{QUIT} =
7360         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7361
7362     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7363     POSIX::mkfifo($mapfifo, 0600);
7364     my $fifofh;
7365     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7366
7367     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7368
7369     my $oldtimeout;
7370     my $timeout = 5;
7371
7372     my $devinfo = {}; # info about drives included in backup
7373     my $virtdev_hash = {}; # info about allocated drives
7374
7375     my $rpcenv = PVE::RPCEnvironment::get();
7376
7377     my $conffile = PVE::QemuConfig->config_file($vmid);
7378
7379     # Note: $oldconf is undef if VM does not exist
7380     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7381     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7382     my $new_conf_raw = '';
7383
7384     my %storage_limits;
7385
7386     my $print_devmap = sub {
7387         my $cfgfn = "$tmpdir/qemu-server.conf";
7388
7389         # we can read the config - that is already extracted
7390         my $fh = IO::File->new($cfgfn, "r") ||
7391             die "unable to read qemu-server.conf - $!\n";
7392
7393         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7394         if (-f $fwcfgfn) {
7395             my $pve_firewall_dir = '/etc/pve/firewall';
7396             mkdir $pve_firewall_dir; # make sure the dir exists
7397             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7398         }
7399
7400         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7401
7402         foreach my $info (values %{$virtdev_hash}) {
7403             my $storeid = $info->{storeid};
7404             next if defined($storage_limits{$storeid});
7405
7406             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7407             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7408             $storage_limits{$storeid} = $limit * 1024;
7409         }
7410
7411         foreach my $devname (keys %$devinfo) {
7412             die "found no device mapping information for device '$devname'\n"
7413                 if !$devinfo->{$devname}->{virtdev};
7414         }
7415
7416         # create empty/temp config
7417         if ($oldconf) {
7418             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7419             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7420         }
7421
7422         # allocate volumes
7423         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7424
7425         # print restore information to $fifofh
7426         foreach my $virtdev (sort keys %$virtdev_hash) {
7427             my $d = $virtdev_hash->{$virtdev};
7428             next if $d->{is_cloudinit}; # no need to restore cloudinit
7429
7430             my $storeid = $d->{storeid};
7431             my $volid = $d->{volid};
7432
7433             my $map_opts = '';
7434             if (my $limit = $storage_limits{$storeid}) {
7435                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7436             }
7437
7438             my $write_zeros = 1;
7439             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7440                 $write_zeros = 0;
7441             }
7442
7443             my $path = PVE::Storage::path($cfg, $volid);
7444
7445             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7446
7447             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7448         }
7449
7450         $fh->seek(0, 0) || die "seek failed - $!\n";
7451
7452         my $cookie = { netcount => 0 };
7453         while (defined(my $line = <$fh>)) {
7454             $new_conf_raw .= restore_update_config_line(
7455                 $cookie,
7456                 $map,
7457                 $line,
7458                 $opts->{unique},
7459             );
7460         }
7461
7462         $fh->close();
7463     };
7464
7465     eval {
7466         # enable interrupts
7467         local $SIG{INT} =
7468             local $SIG{TERM} =
7469             local $SIG{QUIT} =
7470             local $SIG{HUP} =
7471             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7472         local $SIG{ALRM} = sub { die "got timeout\n"; };
7473
7474         $oldtimeout = alarm($timeout);
7475
7476         my $parser = sub {
7477             my $line = shift;
7478
7479             print "$line\n";
7480
7481             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7482                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7483                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7484             } elsif ($line =~ m/^CTIME: /) {
7485                 # we correctly received the vma config, so we can disable
7486                 # the timeout now for disk allocation (set to 10 minutes, so
7487                 # that we always timeout if something goes wrong)
7488                 alarm(600);
7489                 &$print_devmap();
7490                 print $fifofh "done\n";
7491                 my $tmp = $oldtimeout || 0;
7492                 $oldtimeout = undef;
7493                 alarm($tmp);
7494                 close($fifofh);
7495                 $fifofh = undef;
7496             }
7497         };
7498
7499         print "restore vma archive: $dbg_cmdstring\n";
7500         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7501     };
7502     my $err = $@;
7503
7504     alarm($oldtimeout) if $oldtimeout;
7505
7506     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7507
7508     close($fifofh) if $fifofh;
7509     unlink $mapfifo;
7510     rmtree $tmpdir;
7511
7512     if ($err) {
7513         $restore_destroy_volumes->($cfg, $virtdev_hash);
7514         die $err;
7515     }
7516
7517     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7518     check_restore_permissions($rpcenv, $user, $new_conf);
7519     PVE::QemuConfig->write_config($vmid, $new_conf);
7520
7521     eval { rescan($vmid, 1); };
7522     warn $@ if $@;
7523
7524     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7525 }
7526
7527 sub restore_tar_archive {
7528     my ($archive, $vmid, $user, $opts) = @_;
7529
7530     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7531         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7532         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7533     }
7534
7535     if ($archive ne '-') {
7536         my $firstfile = tar_archive_read_firstfile($archive);
7537         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7538             if $firstfile ne 'qemu-server.conf';
7539     }
7540
7541     my $storecfg = PVE::Storage::config();
7542
7543     # avoid zombie disks when restoring over an existing VM -> cleanup first
7544     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7545     # skiplock=1 because qmrestore has set the 'create' lock itself already
7546     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7547     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7548
7549     my $tocmd = "/usr/lib/qemu-server/qmextract";
7550
7551     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7552     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7553     $tocmd .= ' --prealloc' if $opts->{prealloc};
7554     $tocmd .= ' --info' if $opts->{info};
7555
7556     # tar option "xf" does not autodetect compression when read from STDIN,
7557     # so we pipe to zcat
7558     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7559         PVE::Tools::shellquote("--to-command=$tocmd");
7560
7561     my $tmpdir = "/var/tmp/vzdumptmp$$";
7562     mkpath $tmpdir;
7563
7564     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7565     local $ENV{VZDUMP_VMID} = $vmid;
7566     local $ENV{VZDUMP_USER} = $user;
7567
7568     my $conffile = PVE::QemuConfig->config_file($vmid);
7569     my $new_conf_raw = '';
7570
7571     # disable interrupts (always do cleanups)
7572     local $SIG{INT} =
7573         local $SIG{TERM} =
7574         local $SIG{QUIT} =
7575         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7576
7577     eval {
7578         # enable interrupts
7579         local $SIG{INT} =
7580             local $SIG{TERM} =
7581             local $SIG{QUIT} =
7582             local $SIG{HUP} =
7583             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7584
7585         if ($archive eq '-') {
7586             print "extracting archive from STDIN\n";
7587             run_command($cmd, input => "<&STDIN");
7588         } else {
7589             print "extracting archive '$archive'\n";
7590             run_command($cmd);
7591         }
7592
7593         return if $opts->{info};
7594
7595         # read new mapping
7596         my $map = {};
7597         my $statfile = "$tmpdir/qmrestore.stat";
7598         if (my $fd = IO::File->new($statfile, "r")) {
7599             while (defined (my $line = <$fd>)) {
7600                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7601                     $map->{$1} = $2 if $1;
7602                 } else {
7603                     print STDERR "unable to parse line in statfile - $line\n";
7604                 }
7605             }
7606             $fd->close();
7607         }
7608
7609         my $confsrc = "$tmpdir/qemu-server.conf";
7610
7611         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7612
7613         my $cookie = { netcount => 0 };
7614         while (defined (my $line = <$srcfd>)) {
7615             $new_conf_raw .= restore_update_config_line(
7616                 $cookie,
7617                 $map,
7618                 $line,
7619                 $opts->{unique},
7620             );
7621         }
7622
7623         $srcfd->close();
7624     };
7625     if (my $err = $@) {
7626         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7627         die $err;
7628     }
7629
7630     rmtree $tmpdir;
7631
7632     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7633
7634     PVE::Cluster::cfs_update(); # make sure we read new file
7635
7636     eval { rescan($vmid, 1); };
7637     warn $@ if $@;
7638 };
7639
7640 sub foreach_storage_used_by_vm {
7641     my ($conf, $func) = @_;
7642
7643     my $sidhash = {};
7644
7645     PVE::QemuConfig->foreach_volume($conf, sub {
7646         my ($ds, $drive) = @_;
7647         return if drive_is_cdrom($drive);
7648
7649         my $volid = $drive->{file};
7650
7651         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7652         $sidhash->{$sid} = $sid if $sid;
7653     });
7654
7655     foreach my $sid (sort keys %$sidhash) {
7656         &$func($sid);
7657     }
7658 }
7659
7660 my $qemu_snap_storage = {
7661     rbd => 1,
7662 };
7663 sub do_snapshots_with_qemu {
7664     my ($storecfg, $volid, $deviceid) = @_;
7665
7666     return if $deviceid =~ m/tpmstate0/;
7667
7668     my $storage_name = PVE::Storage::parse_volume_id($volid);
7669     my $scfg = $storecfg->{ids}->{$storage_name};
7670     die "could not find storage '$storage_name'\n" if !defined($scfg);
7671
7672     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7673         return 1;
7674     }
7675
7676     if ($volid =~ m/\.(qcow2|qed)$/){
7677         return 1;
7678     }
7679
7680     return;
7681 }
7682
7683 sub qga_check_running {
7684     my ($vmid, $nowarn) = @_;
7685
7686     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7687     if ($@) {
7688         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7689         return 0;
7690     }
7691     return 1;
7692 }
7693
7694 sub template_create {
7695     my ($vmid, $conf, $disk) = @_;
7696
7697     my $storecfg = PVE::Storage::config();
7698
7699     PVE::QemuConfig->foreach_volume($conf, sub {
7700         my ($ds, $drive) = @_;
7701
7702         return if drive_is_cdrom($drive);
7703         return if $disk && $ds ne $disk;
7704
7705         my $volid = $drive->{file};
7706         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7707
7708         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7709         $drive->{file} = $voliddst;
7710         $conf->{$ds} = print_drive($drive);
7711         PVE::QemuConfig->write_config($vmid, $conf);
7712     });
7713 }
7714
7715 sub convert_iscsi_path {
7716     my ($path) = @_;
7717
7718     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7719         my $portal = $1;
7720         my $target = $2;
7721         my $lun = $3;
7722
7723         my $initiator_name = get_initiator_name();
7724
7725         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7726                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7727     }
7728
7729     die "cannot convert iscsi path '$path', unkown format\n";
7730 }
7731
7732 sub qemu_img_convert {
7733     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7734
7735     my $storecfg = PVE::Storage::config();
7736     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7737     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7738
7739     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7740
7741     my $cachemode;
7742     my $src_path;
7743     my $src_is_iscsi = 0;
7744     my $src_format;
7745
7746     if ($src_storeid) {
7747         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7748         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7749         $src_format = qemu_img_format($src_scfg, $src_volname);
7750         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7751         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7752         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7753     } elsif (-f $src_volid || -b $src_volid) {
7754         $src_path = $src_volid;
7755         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7756             $src_format = $1;
7757         }
7758     }
7759
7760     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7761
7762     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7763     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7764     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7765     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7766
7767     my $cmd = [];
7768     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7769     push @$cmd, '-l', "snapshot.name=$snapname"
7770         if $snapname && $src_format && $src_format eq "qcow2";
7771     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7772     push @$cmd, '-T', $cachemode if defined($cachemode);
7773     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7774
7775     if ($src_is_iscsi) {
7776         push @$cmd, '--image-opts';
7777         $src_path = convert_iscsi_path($src_path);
7778     } elsif ($src_format) {
7779         push @$cmd, '-f', $src_format;
7780     }
7781
7782     if ($dst_is_iscsi) {
7783         push @$cmd, '--target-image-opts';
7784         $dst_path = convert_iscsi_path($dst_path);
7785     } else {
7786         push @$cmd, '-O', $dst_format;
7787     }
7788
7789     push @$cmd, $src_path;
7790
7791     if (!$dst_is_iscsi && $is_zero_initialized) {
7792         push @$cmd, "zeroinit:$dst_path";
7793     } else {
7794         push @$cmd, $dst_path;
7795     }
7796
7797     my $parser = sub {
7798         my $line = shift;
7799         if($line =~ m/\((\S+)\/100\%\)/){
7800             my $percent = $1;
7801             my $transferred = int($size * $percent / 100);
7802             my $total_h = render_bytes($size, 1);
7803             my $transferred_h = render_bytes($transferred, 1);
7804
7805             print "transferred $transferred_h of $total_h ($percent%)\n";
7806         }
7807
7808     };
7809
7810     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7811     my $err = $@;
7812     die "copy failed: $err" if $err;
7813 }
7814
7815 sub qemu_img_format {
7816     my ($scfg, $volname) = @_;
7817
7818     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7819         return $1;
7820     } else {
7821         return "raw";
7822     }
7823 }
7824
7825 sub qemu_drive_mirror {
7826     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7827
7828     $jobs = {} if !$jobs;
7829
7830     my $qemu_target;
7831     my $format;
7832     $jobs->{"drive-$drive"} = {};
7833
7834     if ($dst_volid =~ /^nbd:/) {
7835         $qemu_target = $dst_volid;
7836         $format = "nbd";
7837     } else {
7838         my $storecfg = PVE::Storage::config();
7839         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7840
7841         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7842
7843         $format = qemu_img_format($dst_scfg, $dst_volname);
7844
7845         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7846
7847         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7848     }
7849
7850     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7851     $opts->{format} = $format if $format;
7852
7853     if (defined($src_bitmap)) {
7854         $opts->{sync} = 'incremental';
7855         $opts->{bitmap} = $src_bitmap;
7856         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7857     }
7858
7859     if (defined($bwlimit)) {
7860         $opts->{speed} = $bwlimit * 1024;
7861         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7862     } else {
7863         print "drive mirror is starting for drive-$drive\n";
7864     }
7865
7866     # if a job already runs for this device we get an error, catch it for cleanup
7867     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7868     if (my $err = $@) {
7869         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7870         warn "$@\n" if $@;
7871         die "mirroring error: $err\n";
7872     }
7873
7874     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7875 }
7876
7877 # $completion can be either
7878 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7879 # 'cancel': wait until all jobs are ready, block-job-cancel them
7880 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7881 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7882 sub qemu_drive_mirror_monitor {
7883     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7884
7885     $completion //= 'complete';
7886     $op //= "mirror";
7887
7888     eval {
7889         my $err_complete = 0;
7890
7891         my $starttime = time ();
7892         while (1) {
7893             die "block job ('$op') timed out\n" if $err_complete > 300;
7894
7895             my $stats = mon_cmd($vmid, "query-block-jobs");
7896             my $ctime = time();
7897
7898             my $running_jobs = {};
7899             for my $stat (@$stats) {
7900                 next if $stat->{type} ne $op;
7901                 $running_jobs->{$stat->{device}} = $stat;
7902             }
7903
7904             my $readycounter = 0;
7905
7906             for my $job_id (sort keys %$jobs) {
7907                 my $job = $running_jobs->{$job_id};
7908
7909                 my $vanished = !defined($job);
7910                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7911                 if($complete || ($vanished && $completion eq 'auto')) {
7912                     print "$job_id: $op-job finished\n";
7913                     delete $jobs->{$job_id};
7914                     next;
7915                 }
7916
7917                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7918
7919                 my $busy = $job->{busy};
7920                 my $ready = $job->{ready};
7921                 if (my $total = $job->{len}) {
7922                     my $transferred = $job->{offset} || 0;
7923                     my $remaining = $total - $transferred;
7924                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7925
7926                     my $duration = $ctime - $starttime;
7927                     my $total_h = render_bytes($total, 1);
7928                     my $transferred_h = render_bytes($transferred, 1);
7929
7930                     my $status = sprintf(
7931                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7932                         render_duration($duration),
7933                     );
7934
7935                     if ($ready) {
7936                         if ($busy) {
7937                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7938                         } else {
7939                             $status .= ", ready";
7940                         }
7941                     }
7942                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7943                     $jobs->{$job_id}->{ready} = $ready;
7944                 }
7945
7946                 $readycounter++ if $job->{ready};
7947             }
7948
7949             last if scalar(keys %$jobs) == 0;
7950
7951             if ($readycounter == scalar(keys %$jobs)) {
7952                 print "all '$op' jobs are ready\n";
7953
7954                 # do the complete later (or has already been done)
7955                 last if $completion eq 'skip' || $completion eq 'auto';
7956
7957                 if ($vmiddst && $vmiddst != $vmid) {
7958                     my $agent_running = $qga && qga_check_running($vmid);
7959                     if ($agent_running) {
7960                         print "freeze filesystem\n";
7961                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7962                         warn $@ if $@;
7963                     } else {
7964                         print "suspend vm\n";
7965                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7966                         warn $@ if $@;
7967                     }
7968
7969                     # if we clone a disk for a new target vm, we don't switch the disk
7970                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7971
7972                     if ($agent_running) {
7973                         print "unfreeze filesystem\n";
7974                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7975                         warn $@ if $@;
7976                     } else {
7977                         print "resume vm\n";
7978                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7979                         warn $@ if $@;
7980                     }
7981
7982                     last;
7983                 } else {
7984
7985                     for my $job_id (sort keys %$jobs) {
7986                         # try to switch the disk if source and destination are on the same guest
7987                         print "$job_id: Completing block job_id...\n";
7988
7989                         my $op;
7990                         if ($completion eq 'complete') {
7991                             $op = 'block-job-complete';
7992                         } elsif ($completion eq 'cancel') {
7993                             $op = 'block-job-cancel';
7994                         } else {
7995                             die "invalid completion value: $completion\n";
7996                         }
7997                         eval { mon_cmd($vmid, $op, device => $job_id) };
7998                         if ($@ =~ m/cannot be completed/) {
7999                             print "$job_id: block job cannot be completed, trying again.\n";
8000                             $err_complete++;
8001                         }else {
8002                             print "$job_id: Completed successfully.\n";
8003                             $jobs->{$job_id}->{complete} = 1;
8004                         }
8005                     }
8006                 }
8007             }
8008             sleep 1;
8009         }
8010     };
8011     my $err = $@;
8012
8013     if ($err) {
8014         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
8015         die "block job ($op) error: $err";
8016     }
8017 }
8018
8019 sub qemu_blockjobs_cancel {
8020     my ($vmid, $jobs) = @_;
8021
8022     foreach my $job (keys %$jobs) {
8023         print "$job: Cancelling block job\n";
8024         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
8025         $jobs->{$job}->{cancel} = 1;
8026     }
8027
8028     while (1) {
8029         my $stats = mon_cmd($vmid, "query-block-jobs");
8030
8031         my $running_jobs = {};
8032         foreach my $stat (@$stats) {
8033             $running_jobs->{$stat->{device}} = $stat;
8034         }
8035
8036         foreach my $job (keys %$jobs) {
8037
8038             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
8039                 print "$job: Done.\n";
8040                 delete $jobs->{$job};
8041             }
8042         }
8043
8044         last if scalar(keys %$jobs) == 0;
8045
8046         sleep 1;
8047     }
8048 }
8049
8050 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8051 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8052 my sub clone_disk_check_io_uring {
8053     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8054
8055     return if !$use_drive_mirror;
8056
8057     # Don't complain when not changing storage.
8058     # Assume if it works for the source, it'll work for the target too.
8059     return if $src_storeid eq $dst_storeid;
8060
8061     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8062     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8063
8064     my $cache_direct = drive_uses_cache_direct($src_drive);
8065
8066     my $src_uses_io_uring;
8067     if ($src_drive->{aio}) {
8068         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8069     } else {
8070         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8071     }
8072
8073     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8074         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8075 }
8076
8077 sub clone_disk {
8078     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8079
8080     my ($vmid, $running) = $source->@{qw(vmid running)};
8081     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8082
8083     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8084     my ($storage, $format) = $dest->@{qw(storage format)};
8085
8086     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8087
8088     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8089         die "cloning from/to EFI disk requires EFI disk\n"
8090             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8091         die "cloning from/to TPM state requires TPM state\n"
8092             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8093
8094         # This would lead to two device nodes in QEMU pointing to the same backing image!
8095         die "cannot change drive name when cloning disk from/to the same VM\n"
8096             if $use_drive_mirror && $vmid == $newvmid;
8097     }
8098
8099     die "cannot move TPM state while VM is running\n"
8100         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8101
8102     my $newvolid;
8103
8104     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8105     print "$src_drivename " if $src_drivename;
8106     print "($drive->{file})\n";
8107
8108     if (!$full) {
8109         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8110         push @$newvollist, $newvolid;
8111     } else {
8112         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8113         my $storeid = $storage || $src_storeid;
8114
8115         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8116
8117         my $name = undef;
8118         my $size = undef;
8119         if (drive_is_cloudinit($drive)) {
8120             $name = "vm-$newvmid-cloudinit";
8121             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8122             if ($scfg->{path}) {
8123                 $name .= ".$dst_format";
8124             }
8125             $snapname = undef;
8126             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8127         } elsif ($dst_drivename eq 'efidisk0') {
8128             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8129         } elsif ($dst_drivename eq 'tpmstate0') {
8130             $dst_format = 'raw';
8131             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8132         } else {
8133             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8134
8135             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8136         }
8137         $newvolid = PVE::Storage::vdisk_alloc(
8138             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8139         );
8140         push @$newvollist, $newvolid;
8141
8142         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8143
8144         if (drive_is_cloudinit($drive)) {
8145             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8146             # if this is the case, we have to complete any block-jobs still there from
8147             # previous drive-mirrors
8148             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8149                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8150             }
8151             goto no_data_clone;
8152         }
8153
8154         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8155         if ($use_drive_mirror) {
8156             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8157                 $completion, $qga, $bwlimit);
8158         } else {
8159             if ($dst_drivename eq 'efidisk0') {
8160                 # the relevant data on the efidisk may be smaller than the source
8161                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8162                 # that is given by the OVMF_VARS.fd
8163                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8164                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8165
8166                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8167
8168                 # better for Ceph if block size is not too small, see bug #3324
8169                 my $bs = 1024*1024;
8170
8171                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8172
8173                 if ($src_format eq 'qcow2' && $snapname) {
8174                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8175                         if !min_version(kvm_user_version(), 6, 2);
8176                     push $cmd->@*, '-l', $snapname;
8177                 }
8178                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8179                 run_command($cmd);
8180             } else {
8181                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8182             }
8183         }
8184     }
8185
8186 no_data_clone:
8187     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8188
8189     my $disk = dclone($drive);
8190     delete $disk->{format};
8191     $disk->{file} = $newvolid;
8192     $disk->{size} = $size if defined($size);
8193
8194     return $disk;
8195 }
8196
8197 sub get_running_qemu_version {
8198     my ($vmid) = @_;
8199     my $res = mon_cmd($vmid, "query-version");
8200     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8201 }
8202
8203 sub qemu_use_old_bios_files {
8204     my ($machine_type) = @_;
8205
8206     return if !$machine_type;
8207
8208     my $use_old_bios_files = undef;
8209
8210     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8211         $machine_type = $1;
8212         $use_old_bios_files = 1;
8213     } else {
8214         my $version = extract_version($machine_type, kvm_user_version());
8215         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8216         # load new efi bios files on migration. So this hack is required to allow
8217         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8218         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8219         $use_old_bios_files = !min_version($version, 2, 4);
8220     }
8221
8222     return ($use_old_bios_files, $machine_type);
8223 }
8224
8225 sub get_efivars_size {
8226     my ($conf, $efidisk) = @_;
8227
8228     my $arch = get_vm_arch($conf);
8229     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8230     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8231     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8232     return -s $ovmf_vars;
8233 }
8234
8235 sub update_efidisk_size {
8236     my ($conf) = @_;
8237
8238     return if !defined($conf->{efidisk0});
8239
8240     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8241     $disk->{size} = get_efivars_size($conf);
8242     $conf->{efidisk0} = print_drive($disk);
8243
8244     return;
8245 }
8246
8247 sub update_tpmstate_size {
8248     my ($conf) = @_;
8249
8250     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8251     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8252     $conf->{tpmstate0} = print_drive($disk);
8253 }
8254
8255 sub create_efidisk($$$$$$$) {
8256     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8257
8258     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8259
8260     my $vars_size_b = -s $ovmf_vars;
8261     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8262     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8263     PVE::Storage::activate_volumes($storecfg, [$volid]);
8264
8265     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8266     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8267
8268     return ($volid, $size/1024);
8269 }
8270
8271 sub vm_iothreads_list {
8272     my ($vmid) = @_;
8273
8274     my $res = mon_cmd($vmid, 'query-iothreads');
8275
8276     my $iothreads = {};
8277     foreach my $iothread (@$res) {
8278         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8279     }
8280
8281     return $iothreads;
8282 }
8283
8284 sub scsihw_infos {
8285     my ($conf, $drive) = @_;
8286
8287     my $maxdev = 0;
8288
8289     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8290         $maxdev = 7;
8291     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8292         $maxdev = 1;
8293     } else {
8294         $maxdev = 256;
8295     }
8296
8297     my $controller = int($drive->{index} / $maxdev);
8298     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8299         ? "virtioscsi"
8300         : "scsihw";
8301
8302     return ($maxdev, $controller, $controller_prefix);
8303 }
8304
8305 sub resolve_dst_disk_format {
8306         my ($storecfg, $storeid, $src_volname, $format) = @_;
8307         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8308
8309         if (!$format) {
8310             # if no target format is specified, use the source disk format as hint
8311             if ($src_volname) {
8312                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8313                 $format = qemu_img_format($scfg, $src_volname);
8314             } else {
8315                 return $defFormat;
8316             }
8317         }
8318
8319         # test if requested format is supported - else use default
8320         my $supported = grep { $_ eq $format } @$validFormats;
8321         $format = $defFormat if !$supported;
8322         return $format;
8323 }
8324
8325 # NOTE: if this logic changes, please update docs & possibly gui logic
8326 sub find_vmstate_storage {
8327     my ($conf, $storecfg) = @_;
8328
8329     # first, return storage from conf if set
8330     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8331
8332     my ($target, $shared, $local);
8333
8334     foreach_storage_used_by_vm($conf, sub {
8335         my ($sid) = @_;
8336         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8337         my $dst = $scfg->{shared} ? \$shared : \$local;
8338         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8339     });
8340
8341     # second, use shared storage where VM has at least one disk
8342     # third, use local storage where VM has at least one disk
8343     # fall back to local storage
8344     $target = $shared // $local // 'local';
8345
8346     return $target;
8347 }
8348
8349 sub generate_uuid {
8350     my ($uuid, $uuid_str);
8351     UUID::generate($uuid);
8352     UUID::unparse($uuid, $uuid_str);
8353     return $uuid_str;
8354 }
8355
8356 sub generate_smbios1_uuid {
8357     return "uuid=".generate_uuid();
8358 }
8359
8360 sub nbd_stop {
8361     my ($vmid) = @_;
8362
8363     mon_cmd($vmid, 'nbd-server-stop');
8364 }
8365
8366 sub create_reboot_request {
8367     my ($vmid) = @_;
8368     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8369         or die "failed to create reboot trigger file: $!\n";
8370     close($fh);
8371 }
8372
8373 sub clear_reboot_request {
8374     my ($vmid) = @_;
8375     my $path = "/run/qemu-server/$vmid.reboot";
8376     my $res = 0;
8377
8378     $res = unlink($path);
8379     die "could not remove reboot request for $vmid: $!"
8380         if !$res && $! != POSIX::ENOENT;
8381
8382     return $res;
8383 }
8384
8385 sub bootorder_from_legacy {
8386     my ($conf, $bootcfg) = @_;
8387
8388     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8389     my $bootindex_hash = {};
8390     my $i = 1;
8391     foreach my $o (split(//, $boot)) {
8392         $bootindex_hash->{$o} = $i*100;
8393         $i++;
8394     }
8395
8396     my $bootorder = {};
8397
8398     PVE::QemuConfig->foreach_volume($conf, sub {
8399         my ($ds, $drive) = @_;
8400
8401         if (drive_is_cdrom ($drive, 1)) {
8402             if ($bootindex_hash->{d}) {
8403                 $bootorder->{$ds} = $bootindex_hash->{d};
8404                 $bootindex_hash->{d} += 1;
8405             }
8406         } elsif ($bootindex_hash->{c}) {
8407             $bootorder->{$ds} = $bootindex_hash->{c}
8408                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8409             $bootindex_hash->{c} += 1;
8410         }
8411     });
8412
8413     if ($bootindex_hash->{n}) {
8414         for (my $i = 0; $i < $MAX_NETS; $i++) {
8415             my $netname = "net$i";
8416             next if !$conf->{$netname};
8417             $bootorder->{$netname} = $bootindex_hash->{n};
8418             $bootindex_hash->{n} += 1;
8419         }
8420     }
8421
8422     return $bootorder;
8423 }
8424
8425 # Generate default device list for 'boot: order=' property. Matches legacy
8426 # default boot order, but with explicit device names. This is important, since
8427 # the fallback for when neither 'order' nor the old format is specified relies
8428 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8429 sub get_default_bootdevices {
8430     my ($conf) = @_;
8431
8432     my @ret = ();
8433
8434     # harddisk
8435     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8436     push @ret, $first if $first;
8437
8438     # cdrom
8439     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8440     push @ret, $first if $first;
8441
8442     # network
8443     for (my $i = 0; $i < $MAX_NETS; $i++) {
8444         my $netname = "net$i";
8445         next if !$conf->{$netname};
8446         push @ret, $netname;
8447         last;
8448     }
8449
8450     return \@ret;
8451 }
8452
8453 sub device_bootorder {
8454     my ($conf) = @_;
8455
8456     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8457
8458     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8459
8460     my $bootorder = {};
8461     if (!defined($boot) || $boot->{legacy}) {
8462         $bootorder = bootorder_from_legacy($conf, $boot);
8463     } elsif ($boot->{order}) {
8464         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8465         for my $dev (PVE::Tools::split_list($boot->{order})) {
8466             $bootorder->{$dev} = $i++;
8467         }
8468     }
8469
8470     return $bootorder;
8471 }
8472
8473 sub register_qmeventd_handle {
8474     my ($vmid) = @_;
8475
8476     my $fh;
8477     my $peer = "/var/run/qmeventd.sock";
8478     my $count = 0;
8479
8480     for (;;) {
8481         $count++;
8482         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8483         last if $fh;
8484         if ($! != EINTR && $! != EAGAIN) {
8485             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8486         }
8487         if ($count > 4) {
8488             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8489               . "after $count retries\n";
8490         }
8491         usleep(25000);
8492     }
8493
8494     # send handshake to mark VM as backing up
8495     print $fh to_json({vzdump => {vmid => "$vmid"}});
8496
8497     # return handle to be closed later when inhibit is no longer required
8498     return $fh;
8499 }
8500
8501 # bash completion helper
8502
8503 sub complete_backup_archives {
8504     my ($cmdname, $pname, $cvalue) = @_;
8505
8506     my $cfg = PVE::Storage::config();
8507
8508     my $storeid;
8509
8510     if ($cvalue =~ m/^([^:]+):/) {
8511         $storeid = $1;
8512     }
8513
8514     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8515
8516     my $res = [];
8517     foreach my $id (keys %$data) {
8518         foreach my $item (@{$data->{$id}}) {
8519             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8520             push @$res, $item->{volid} if defined($item->{volid});
8521         }
8522     }
8523
8524     return $res;
8525 }
8526
8527 my $complete_vmid_full = sub {
8528     my ($running) = @_;
8529
8530     my $idlist = vmstatus();
8531
8532     my $res = [];
8533
8534     foreach my $id (keys %$idlist) {
8535         my $d = $idlist->{$id};
8536         if (defined($running)) {
8537             next if $d->{template};
8538             next if $running && $d->{status} ne 'running';
8539             next if !$running && $d->{status} eq 'running';
8540         }
8541         push @$res, $id;
8542
8543     }
8544     return $res;
8545 };
8546
8547 sub complete_vmid {
8548     return &$complete_vmid_full();
8549 }
8550
8551 sub complete_vmid_stopped {
8552     return &$complete_vmid_full(0);
8553 }
8554
8555 sub complete_vmid_running {
8556     return &$complete_vmid_full(1);
8557 }
8558
8559 sub complete_storage {
8560
8561     my $cfg = PVE::Storage::config();
8562     my $ids = $cfg->{ids};
8563
8564     my $res = [];
8565     foreach my $sid (keys %$ids) {
8566         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8567         next if !$ids->{$sid}->{content}->{images};
8568         push @$res, $sid;
8569     }
8570
8571     return $res;
8572 }
8573
8574 sub complete_migration_storage {
8575     my ($cmd, $param, $current_value, $all_args) = @_;
8576
8577     my $targetnode = @$all_args[1];
8578
8579     my $cfg = PVE::Storage::config();
8580     my $ids = $cfg->{ids};
8581
8582     my $res = [];
8583     foreach my $sid (keys %$ids) {
8584         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8585         next if !$ids->{$sid}->{content}->{images};
8586         push @$res, $sid;
8587     }
8588
8589     return $res;
8590 }
8591
8592 sub vm_is_paused {
8593     my ($vmid) = @_;
8594     my $qmpstatus = eval {
8595         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8596         mon_cmd($vmid, "query-status");
8597     };
8598     warn "$@\n" if $@;
8599     return $qmpstatus && $qmpstatus->{status} eq "paused";
8600 }
8601
8602 sub check_volume_storage_type {
8603     my ($storecfg, $vol) = @_;
8604
8605     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8606     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8607     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8608
8609     die "storage '$storeid' does not support content-type '$vtype'\n"
8610         if !$scfg->{content}->{$vtype};
8611
8612     return 1;
8613 }
8614
8615 sub add_nets_bridge_fdb {
8616     my ($conf, $vmid) = @_;
8617
8618     for my $opt (keys %$conf) {
8619         next if $opt !~ m/^net(\d+)$/;
8620         my $iface = "tap${vmid}i$1";
8621         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8622         my $net = parse_net($conf->{$opt}, 1) or next;
8623
8624         my $mac = $net->{macaddr};
8625         if (!$mac) {
8626             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8627                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8628             next;
8629         }
8630
8631         my $bridge = $net->{bridge};
8632         if (!$bridge) {
8633             log_warn("Interface '$iface' not attached to any bridge.");
8634             next;
8635         }
8636         if ($have_sdn) {
8637             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8638         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8639             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8640         }
8641     }
8642 }
8643
8644 sub del_nets_bridge_fdb {
8645     my ($conf, $vmid) = @_;
8646
8647     for my $opt (keys %$conf) {
8648         next if $opt !~ m/^net(\d+)$/;
8649         my $iface = "tap${vmid}i$1";
8650
8651         my $net = parse_net($conf->{$opt}) or next;
8652         my $mac = $net->{macaddr} or next;
8653
8654         my $bridge = $net->{bridge};
8655         if ($have_sdn) {
8656             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8657         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8658             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8659         }
8660     }
8661 }
8662
8663 1;