PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory;
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::USB;
  62
  63 my $have_sdn;
  64 eval {
  65     require PVE::Network::SDN::Zones;
  66     $have_sdn = 1;
  67 };
  68
  69 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  70 my $OVMF = {
  71     x86_64 => {
  72         '4m-no-smm' => [
  73             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  74             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  75         ],
  76         '4m-no-smm-ms' => [
  77             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  78             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  79         ],
  80         '4m' => [
  81             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  82             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  83         ],
  84         '4m-ms' => [
  85             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  86             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  87         ],
  88         default => [
  89             "$EDK2_FW_BASE/OVMF_CODE.fd",
  90             "$EDK2_FW_BASE/OVMF_VARS.fd",
  91         ],
  92     },
  93     aarch64 => {
  94         default => [
  95             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  96             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  97         ],
  98     },
  99 };
 100
 101 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 102
 103 # Note about locking: we use flock on the config file protect against concurent actions.
 104 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 105 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 106 # But you can ignore this kind of lock with the --skiplock flag.
 107
 108 cfs_register_file(
 109     '/qemu-server/',
 110     \&parse_vm_config,
 111     \&write_vm_config
 112 );
 113
 114 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 115     description => "Some command save/restore state from this location.",
 116     type => 'string',
 117     maxLength => 128,
 118     optional => 1,
 119 });
 120
 121 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 122         description => "Specifies the QEMU machine type.",
 123         type => 'string',
 124         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 125         maxLength => 40,
 126         optional => 1,
 127 });
 128
 129 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 130 my $nodename_cache;
 131 sub nodename {
 132     $nodename_cache //= PVE::INotify::nodename();
 133     return $nodename_cache;
 134 }
 135
 136 my $watchdog_fmt = {
 137     model => {
 138         default_key => 1,
 139         type => 'string',
 140         enum => [qw(i6300esb ib700)],
 141         description => "Watchdog type to emulate.",
 142         default => 'i6300esb',
 143         optional => 1,
 144     },
 145     action => {
 146         type => 'string',
 147         enum => [qw(reset shutdown poweroff pause debug none)],
 148         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 149         optional => 1,
 150     },
 151 };
 152 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 153
 154 my $agent_fmt = {
 155     enabled => {
 156         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 157         type => 'boolean',
 158         default => 0,
 159         default_key => 1,
 160     },
 161     fstrim_cloned_disks => {
 162         description => "Run fstrim after moving a disk or migrating the VM.",
 163         type => 'boolean',
 164         optional => 1,
 165         default => 0,
 166     },
 167     'freeze-fs-on-backup' => {
 168         description => "Freeze/thaw guest filesystems on backup for consistency.",
 169         type => 'boolean',
 170         optional => 1,
 171         default => 1,
 172     },
 173     type => {
 174         description => "Select the agent type",
 175         type => 'string',
 176         default => 'virtio',
 177         optional => 1,
 178         enum => [qw(virtio isa)],
 179     },
 180 };
 181
 182 my $vga_fmt = {
 183     type => {
 184         description => "Select the VGA type.",
 185         type => 'string',
 186         default => 'std',
 187         optional => 1,
 188         default_key => 1,
 189         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 190     },
 191     memory => {
 192         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 193         type => 'integer',
 194         optional => 1,
 195         minimum => 4,
 196         maximum => 512,
 197     },
 198 };
 199
 200 my $ivshmem_fmt = {
 201     size => {
 202         type => 'integer',
 203         minimum => 1,
 204         description => "The size of the file in MB.",
 205     },
 206     name => {
 207         type => 'string',
 208         pattern => '[a-zA-Z0-9\-]+',
 209         optional => 1,
 210         format_description => 'string',
 211         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 212     },
 213 };
 214
 215 my $audio_fmt = {
 216     device => {
 217         type => 'string',
 218         enum => [qw(ich9-intel-hda intel-hda AC97)],
 219         description =>  "Configure an audio device."
 220     },
 221     driver =>  {
 222         type => 'string',
 223         enum => ['spice', 'none'],
 224         default => 'spice',
 225         optional => 1,
 226         description => "Driver backend for the audio device."
 227     },
 228 };
 229
 230 my $spice_enhancements_fmt = {
 231     foldersharing => {
 232         type => 'boolean',
 233         optional => 1,
 234         default => '0',
 235         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 236     },
 237     videostreaming =>  {
 238         type => 'string',
 239         enum => ['off', 'all', 'filter'],
 240         default => 'off',
 241         optional => 1,
 242         description => "Enable video streaming. Uses compression for detected video streams."
 243     },
 244 };
 245
 246 my $rng_fmt = {
 247     source => {
 248         type => 'string',
 249         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 250         default_key => 1,
 251         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 252             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 253             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 254             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 255             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 256             ." a hardware RNG from the host.",
 257     },
 258     max_bytes => {
 259         type => 'integer',
 260         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 261             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 262             ." `0` to disable limiting (potentially dangerous!).",
 263         optional => 1,
 264
 265         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 266         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 267         # reading from /dev/urandom
 268         default => 1024,
 269     },
 270     period => {
 271         type => 'integer',
 272         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 273             ." the guest to retrieve another 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $meta_info_fmt = {
 280     'ctime' => {
 281         type => 'integer',
 282         description => "The guest creation timestamp as UNIX epoch time",
 283         minimum => 0,
 284         optional => 1,
 285     },
 286     'creation-qemu' => {
 287         type => 'string',
 288         description => "The QEMU (machine) version from the time this VM was created.",
 289         pattern => '\d+(\.\d+)+',
 290         optional => 1,
 291     },
 292 };
 293
 294 my $confdesc = {
 295     onboot => {
 296         optional => 1,
 297         type => 'boolean',
 298         description => "Specifies whether a VM will be started during system bootup.",
 299         default => 0,
 300     },
 301     autostart => {
 302         optional => 1,
 303         type => 'boolean',
 304         description => "Automatic restart after crash (currently ignored).",
 305         default => 0,
 306     },
 307     hotplug => {
 308         optional => 1,
 309         type => 'string', format => 'pve-hotplug-features',
 310         description => "Selectively enable hotplug features. This is a comma separated list of"
 311             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 312             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 313             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 314             ." windows > 7.",
 315         default => 'network,disk,usb',
 316     },
 317     reboot => {
 318         optional => 1,
 319         type => 'boolean',
 320         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 321         default => 1,
 322     },
 323     lock => {
 324         optional => 1,
 325         type => 'string',
 326         description => "Lock/unlock the VM.",
 327         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 328     },
 329     cpulimit => {
 330         optional => 1,
 331         type => 'number',
 332         description => "Limit of CPU usage.",
 333         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 334             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 335         minimum => 0,
 336         maximum => 128,
 337         default => 0,
 338     },
 339     cpuunits => {
 340         optional => 1,
 341         type => 'integer',
 342         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 343         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 344             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 345             ." weights of all the other running VMs.",
 346         minimum => 1,
 347         maximum => 262144,
 348         default => 'cgroup v1: 1024, cgroup v2: 100',
 349     },
 350     memory => {
 351         optional => 1,
 352         type => 'integer',
 353         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 354             ." you use the balloon device.",
 355         minimum => 16,
 356         default => 512,
 357     },
 358     balloon => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 362         minimum => 0,
 363     },
 364     shares => {
 365         optional => 1,
 366         type => 'integer',
 367         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 368             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 369             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 370         minimum => 0,
 371         maximum => 50000,
 372         default => 1000,
 373     },
 374     keyboard => {
 375         optional => 1,
 376         type => 'string',
 377         description => "Keyboard layout for VNC server. This option is generally not required and"
 378         ." is often better handled from within the guest OS.",
 379         enum => PVE::Tools::kvmkeymaplist(),
 380         default => undef,
 381     },
 382     name => {
 383         optional => 1,
 384         type => 'string', format => 'dns-name',
 385         description => "Set a name for the VM. Only used on the configuration web interface.",
 386     },
 387     scsihw => {
 388         optional => 1,
 389         type => 'string',
 390         description => "SCSI controller model",
 391         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 392         default => 'lsi',
 393     },
 394     description => {
 395         optional => 1,
 396         type => 'string',
 397         description => "Description for the VM. Shown in the web-interface VM's summary."
 398             ." This is saved as comment inside the configuration file.",
 399         maxLength => 1024 * 8,
 400     },
 401     ostype => {
 402         optional => 1,
 403         type => 'string',
 404         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 405         description => "Specify guest operating system.",
 406         verbose_description => <<EODESC,
 407 Specify guest operating system. This is used to enable special
 408 optimization/features for specific operating systems:
 409
 410 [horizontal]
 411 other;; unspecified OS
 412 wxp;; Microsoft Windows XP
 413 w2k;; Microsoft Windows 2000
 414 w2k3;; Microsoft Windows 2003
 415 w2k8;; Microsoft Windows 2008
 416 wvista;; Microsoft Windows Vista
 417 win7;; Microsoft Windows 7
 418 win8;; Microsoft Windows 8/2012/2012r2
 419 win10;; Microsoft Windows 10/2016/2019
 420 win11;; Microsoft Windows 11/2022
 421 l24;; Linux 2.4 Kernel
 422 l26;; Linux 2.6 - 6.X Kernel
 423 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 424 EODESC
 425     },
 426     boot => {
 427         optional => 1,
 428         type => 'string', format => 'pve-qm-boot',
 429         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 430             ." key or 'legacy=' is deprecated.",
 431     },
 432     bootdisk => {
 433         optional => 1,
 434         type => 'string', format => 'pve-qm-bootdisk',
 435         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 436         pattern => '(ide|sata|scsi|virtio)\d+',
 437     },
 438     smp => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPUs. Please use option -sockets instead.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     sockets => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of CPU sockets.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     cores => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of cores per socket.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     numa => {
 460         optional => 1,
 461         type => 'boolean',
 462         description => "Enable/disable NUMA.",
 463         default => 0,
 464     },
 465     hugepages => {
 466         optional => 1,
 467         type => 'string',
 468         description => "Enable/disable hugepages memory.",
 469         enum => [qw(any 2 1024)],
 470     },
 471     keephugepages => {
 472         optional => 1,
 473         type => 'boolean',
 474         default => 0,
 475         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 476             ." after VM shutdown and can be used for subsequent starts.",
 477     },
 478     vcpus => {
 479         optional => 1,
 480         type => 'integer',
 481         description => "Number of hotplugged vcpus.",
 482         minimum => 1,
 483         default => 0,
 484     },
 485     acpi => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Enable/disable ACPI.",
 489         default => 1,
 490     },
 491     agent => {
 492         optional => 1,
 493         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 494         type => 'string',
 495         format => $agent_fmt,
 496     },
 497     kvm => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Enable/disable KVM hardware virtualization.",
 501         default => 1,
 502     },
 503     tdf => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable time drift fix.",
 507         default => 0,
 508     },
 509     localtime => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 513             ." the `ostype` indicates a Microsoft Windows OS.",
 514     },
 515     freeze => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 519     },
 520     vga => {
 521         optional => 1,
 522         type => 'string', format => $vga_fmt,
 523         description => "Configure the VGA hardware.",
 524         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 525             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 526             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 527             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 528             ." display server. For win* OS you can select how many independent displays you want,"
 529             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 530             ." using a serial device as terminal.",
 531     },
 532     watchdog => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-watchdog',
 535         description => "Create a virtual hardware watchdog device.",
 536         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 537             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 538             ." else the watchdog will reset the guest (or execute the respective action specified)",
 539     },
 540     startdate => {
 541         optional => 1,
 542         type => 'string',
 543         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 544         description => "Set the initial date of the real time clock. Valid format for date are:"
 545             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 546         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 547         default => 'now',
 548     },
 549     startup =>  get_standard_option('pve-startup-order'),
 550     template => {
 551         optional => 1,
 552         type => 'boolean',
 553         description => "Enable/disable Template.",
 554         default => 0,
 555     },
 556     args => {
 557         optional => 1,
 558         type => 'string',
 559         description => "Arbitrary arguments passed to kvm.",
 560         verbose_description => <<EODESCR,
 561 Arbitrary arguments passed to kvm, for example:
 562
 563 args: -no-reboot -smbios 'type=0,vendor=FOO'
 564
 565 NOTE: this option is for experts only.
 566 EODESCR
 567     },
 568     tablet => {
 569         optional => 1,
 570         type => 'boolean',
 571         default => 1,
 572         description => "Enable/disable the USB tablet device.",
 573         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 574             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 575             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 576             ." may consider disabling this to save some context switches. This is turned off by"
 577             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 578     },
 579     migrate_speed => {
 580         optional => 1,
 581         type => 'integer',
 582         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 583         minimum => 0,
 584         default => 0,
 585     },
 586     migrate_downtime => {
 587         optional => 1,
 588         type => 'number',
 589         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 590         minimum => 0,
 591         default => 0.1,
 592     },
 593     cdrom => {
 594         optional => 1,
 595         type => 'string', format => 'pve-qm-ide',
 596         typetext => '<volume>',
 597         description => "This is an alias for option -ide2",
 598     },
 599     cpu => {
 600         optional => 1,
 601         description => "Emulated CPU type.",
 602         type => 'string',
 603         format => 'pve-vm-cpu-conf',
 604     },
 605     parent => get_standard_option('pve-snapshot-name', {
 606         optional => 1,
 607         description => "Parent snapshot name. This is used internally, and should not be modified.",
 608     }),
 609     snaptime => {
 610         optional => 1,
 611         description => "Timestamp for snapshots.",
 612         type => 'integer',
 613         minimum => 0,
 614     },
 615     vmstate => {
 616         optional => 1,
 617         type => 'string', format => 'pve-volume-id',
 618         description => "Reference to a volume which stores the VM state. This is used internally"
 619             ." for snapshots.",
 620     },
 621     vmstatestorage => get_standard_option('pve-storage-id', {
 622         description => "Default storage for VM state volumes/files.",
 623         optional => 1,
 624     }),
 625     runningmachine => get_standard_option('pve-qemu-machine', {
 626         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 627             ." for snapshots.",
 628     }),
 629     runningcpu => {
 630         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 631             ." internally for snapshots.",
 632         optional => 1,
 633         type => 'string',
 634         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 635         format_description => 'QEMU -cpu parameter'
 636     },
 637     machine => get_standard_option('pve-qemu-machine'),
 638     arch => {
 639         description => "Virtual processor architecture. Defaults to the host.",
 640         optional => 1,
 641         type => 'string',
 642         enum => [qw(x86_64 aarch64)],
 643     },
 644     smbios1 => {
 645         description => "Specify SMBIOS type 1 fields.",
 646         type => 'string', format => 'pve-qm-smbios1',
 647         maxLength => 512,
 648         optional => 1,
 649     },
 650     protection => {
 651         optional => 1,
 652         type => 'boolean',
 653         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 654             ." remove disk operations.",
 655         default => 0,
 656     },
 657     bios => {
 658         optional => 1,
 659         type => 'string',
 660         enum => [ qw(seabios ovmf) ],
 661         description => "Select BIOS implementation.",
 662         default => 'seabios',
 663     },
 664     vmgenid => {
 665         type => 'string',
 666         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 667         format_description => 'UUID',
 668         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 669             ." to disable explicitly.",
 670         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 671             ." value identifier to the guest OS. This allows to notify the guest operating system"
 672             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 673             ." execution or creation from a template). The guest operating system notices the"
 674             ." change, and is then able to react as appropriate by marking its copies of"
 675             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 676             ."Note that auto-creation only works when done through API/CLI create or update methods"
 677             .", but not when manually editing the config file.",
 678         default => "1 (autogenerated)",
 679         optional => 1,
 680     },
 681     hookscript => {
 682         type => 'string',
 683         format => 'pve-volume-id',
 684         optional => 1,
 685         description => "Script that will be executed during various steps in the vms lifetime.",
 686     },
 687     ivshmem => {
 688         type => 'string',
 689         format => $ivshmem_fmt,
 690         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 691             ." the host.",
 692         optional => 1,
 693     },
 694     audio0 => {
 695         type => 'string',
 696         format => $audio_fmt,
 697         description => "Configure a audio device, useful in combination with QXL/Spice.",
 698         optional => 1
 699     },
 700     spice_enhancements => {
 701         type => 'string',
 702         format => $spice_enhancements_fmt,
 703         description => "Configure additional enhancements for SPICE.",
 704         optional => 1
 705     },
 706     tags => {
 707         type => 'string', format => 'pve-tag-list',
 708         description => 'Tags of the VM. This is only meta information.',
 709         optional => 1,
 710     },
 711     rng0 => {
 712         type => 'string',
 713         format => $rng_fmt,
 714         description => "Configure a VirtIO-based Random Number Generator.",
 715         optional => 1,
 716     },
 717     meta => {
 718         type => 'string',
 719         format => $meta_info_fmt,
 720         description => "Some (read-only) meta-information about this guest.",
 721         optional => 1,
 722     },
 723     affinity => {
 724         type => 'string', format => 'pve-cpuset',
 725         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 726         optional => 1,
 727     },
 728 };
 729
 730 my $cicustom_fmt = {
 731     meta => {
 732         type => 'string',
 733         optional => 1,
 734         description => 'Specify a custom file containing all meta data passed to the VM via"
 735             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 736         format => 'pve-volume-id',
 737         format_description => 'volume',
 738     },
 739     network => {
 740         type => 'string',
 741         optional => 1,
 742         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 743         format => 'pve-volume-id',
 744         format_description => 'volume',
 745     },
 746     user => {
 747         type => 'string',
 748         optional => 1,
 749         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 750         format => 'pve-volume-id',
 751         format_description => 'volume',
 752     },
 753     vendor => {
 754         type => 'string',
 755         optional => 1,
 756         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 757         format => 'pve-volume-id',
 758         format_description => 'volume',
 759     },
 760 };
 761 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 762
 763 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 764 my $confdesc_cloudinit = {
 765     citype => {
 766         optional => 1,
 767         type => 'string',
 768         description => 'Specifies the cloud-init configuration format. The default depends on the'
 769             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 770             .' and `configdrive2` for windows.',
 771         enum => ['configdrive2', 'nocloud', 'opennebula'],
 772     },
 773     ciuser => {
 774         optional => 1,
 775         type => 'string',
 776         description => "cloud-init: User name to change ssh keys and password for instead of the"
 777             ." image's configured default user.",
 778     },
 779     cipassword => {
 780         optional => 1,
 781         type => 'string',
 782         description => 'cloud-init: Password to assign the user. Using this is generally not'
 783             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 784             .' support hashed passwords.',
 785     },
 786     ciupgrade => {
 787         optional => 1,
 788         type => 'boolean',
 789         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 790         default => 1,
 791     },
 792     cicustom => {
 793         optional => 1,
 794         type => 'string',
 795         description => 'cloud-init: Specify custom files to replace the automatically generated'
 796             .' ones at start.',
 797         format => 'pve-qm-cicustom',
 798     },
 799     searchdomain => {
 800         optional => 1,
 801         type => 'string',
 802         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 803             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 804             .' are set.',
 805     },
 806     nameserver => {
 807         optional => 1,
 808         type => 'string', format => 'address-list',
 809         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 810             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 811             .' are set.',
 812     },
 813     sshkeys => {
 814         optional => 1,
 815         type => 'string',
 816         format => 'urlencoded',
 817         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 818     },
 819 };
 820
 821 # what about other qemu settings ?
 822 #cpu => 'string',
 823 #machine => 'string',
 824 #fda => 'file',
 825 #fdb => 'file',
 826 #mtdblock => 'file',
 827 #sd => 'file',
 828 #pflash => 'file',
 829 #snapshot => 'bool',
 830 #bootp => 'file',
 831 ##tftp => 'dir',
 832 ##smb => 'dir',
 833 #kernel => 'file',
 834 #append => 'string',
 835 #initrd => 'file',
 836 ##soundhw => 'string',
 837
 838 while (my ($k, $v) = each %$confdesc) {
 839     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 840 }
 841
 842 my $MAX_NETS = 32;
 843 my $MAX_SERIAL_PORTS = 4;
 844 my $MAX_PARALLEL_PORTS = 3;
 845 my $MAX_NUMA = 8;
 846
 847 my $numa_fmt = {
 848     cpus => {
 849         type => "string",
 850         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 851         description => "CPUs accessing this NUMA node.",
 852         format_description => "id[-id];...",
 853     },
 854     memory => {
 855         type => "number",
 856         description => "Amount of memory this NUMA node provides.",
 857         optional => 1,
 858     },
 859     hostnodes => {
 860         type => "string",
 861         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 862         description => "Host NUMA nodes to use.",
 863         format_description => "id[-id];...",
 864         optional => 1,
 865     },
 866     policy => {
 867         type => 'string',
 868         enum => [qw(preferred bind interleave)],
 869         description => "NUMA allocation policy.",
 870         optional => 1,
 871     },
 872 };
 873 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 874 my $numadesc = {
 875     optional => 1,
 876     type => 'string', format => $numa_fmt,
 877     description => "NUMA topology.",
 878 };
 879 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 880
 881 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 882     $confdesc->{"numa$i"} = $numadesc;
 883 }
 884
 885 my $nic_model_list = [
 886     'e1000',
 887     'e1000-82540em',
 888     'e1000-82544gc',
 889     'e1000-82545em',
 890     'e1000e',
 891     'i82551',
 892     'i82557b',
 893     'i82559er',
 894     'ne2k_isa',
 895     'ne2k_pci',
 896     'pcnet',
 897     'rtl8139',
 898     'virtio',
 899     'vmxnet3',
 900 ];
 901 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 902
 903 my $net_fmt_bridge_descr = <<__EOD__;
 904 Bridge to attach the network device to. The Proxmox VE standard bridge
 905 is called 'vmbr0'.
 906
 907 If you do not specify a bridge, we create a kvm user (NATed) network
 908 device, which provides DHCP and DNS services. The following addresses
 909 are used:
 910
 911  10.0.2.2   Gateway
 912  10.0.2.3   DNS Server
 913  10.0.2.4   SMB Server
 914
 915 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 916 __EOD__
 917
 918 my $net_fmt = {
 919     macaddr  => get_standard_option('mac-addr', {
 920         description => "MAC address. That address must be unique withing your network. This is"
 921             ." automatically generated if not specified.",
 922     }),
 923     model => {
 924         type => 'string',
 925         description => "Network Card Model. The 'virtio' model provides the best performance with"
 926             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 927             ." best to use 'e1000'.",
 928         enum => $nic_model_list,
 929         default_key => 1,
 930     },
 931     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 932     bridge => get_standard_option('pve-bridge-id', {
 933         description => $net_fmt_bridge_descr,
 934         optional => 1,
 935     }),
 936     queues => {
 937         type => 'integer',
 938         minimum => 0, maximum => 64,
 939         description => 'Number of packet queues to be used on the device.',
 940         optional => 1,
 941     },
 942     rate => {
 943         type => 'number',
 944         minimum => 0,
 945         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 946         optional => 1,
 947     },
 948     tag => {
 949         type => 'integer',
 950         minimum => 1, maximum => 4094,
 951         description => 'VLAN tag to apply to packets on this interface.',
 952         optional => 1,
 953     },
 954     trunks => {
 955         type => 'string',
 956         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 957         description => 'VLAN trunks to pass through this interface.',
 958         format_description => 'vlanid[;vlanid...]',
 959         optional => 1,
 960     },
 961     firewall => {
 962         type => 'boolean',
 963         description => 'Whether this interface should be protected by the firewall.',
 964         optional => 1,
 965     },
 966     link_down => {
 967         type => 'boolean',
 968         description => 'Whether this interface should be disconnected (like pulling the plug).',
 969         optional => 1,
 970     },
 971     mtu => {
 972         type => 'integer',
 973         minimum => 1, maximum => 65520,
 974         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 975         optional => 1,
 976     },
 977 };
 978
 979 my $netdesc = {
 980     optional => 1,
 981     type => 'string', format => $net_fmt,
 982     description => "Specify network devices.",
 983 };
 984
 985 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 986
 987 my $ipconfig_fmt = {
 988     ip => {
 989         type => 'string',
 990         format => 'pve-ipv4-config',
 991         format_description => 'IPv4Format/CIDR',
 992         description => 'IPv4 address in CIDR format.',
 993         optional => 1,
 994         default => 'dhcp',
 995     },
 996     gw => {
 997         type => 'string',
 998         format => 'ipv4',
 999         format_description => 'GatewayIPv4',
1000         description => 'Default gateway for IPv4 traffic.',
1001         optional => 1,
1002         requires => 'ip',
1003     },
1004     ip6 => {
1005         type => 'string',
1006         format => 'pve-ipv6-config',
1007         format_description => 'IPv6Format/CIDR',
1008         description => 'IPv6 address in CIDR format.',
1009         optional => 1,
1010         default => 'dhcp',
1011     },
1012     gw6 => {
1013         type => 'string',
1014         format => 'ipv6',
1015         format_description => 'GatewayIPv6',
1016         description => 'Default gateway for IPv6 traffic.',
1017         optional => 1,
1018         requires => 'ip6',
1019     },
1020 };
1021 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1022 my $ipconfigdesc = {
1023     optional => 1,
1024     type => 'string', format => 'pve-qm-ipconfig',
1025     description => <<'EODESCR',
1026 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1027
1028 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1029
1030 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1031 gateway should be provided.
1032 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1033 cloud-init 19.4 or newer.
1034
1035 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1036 dhcp on IPv4.
1037 EODESCR
1038 };
1039 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1040
1041 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1042     $confdesc->{"net$i"} = $netdesc;
1043     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1044 }
1045
1046 foreach my $key (keys %$confdesc_cloudinit) {
1047     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1048 }
1049
1050 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1051 sub pve_verify_cpuset {
1052     my ($set_text, $noerr) = @_;
1053
1054     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1055
1056     if ($@) {
1057         return if $noerr;
1058         die "unable to parse cpuset option\n";
1059     }
1060
1061     return PVE::CpuSet->new($members)->short_string();
1062 }
1063
1064 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1065 sub verify_volume_id_or_qm_path {
1066     my ($volid, $noerr) = @_;
1067
1068     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1069
1070     return verify_volume_id_or_absolute_path($volid, $noerr);
1071 }
1072
1073 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1074 sub verify_volume_id_or_absolute_path {
1075     my ($volid, $noerr) = @_;
1076
1077     return $volid if $volid =~ m|^/|;
1078
1079     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1080     if ($@) {
1081         return if $noerr;
1082         die $@;
1083     }
1084     return $volid;
1085 }
1086
1087 my $serialdesc = {
1088         optional => 1,
1089         type => 'string',
1090         pattern => '(/dev/.+|socket)',
1091         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1092         verbose_description =>  <<EODESCR,
1093 Create a serial device inside the VM (n is 0 to 3), and pass through a
1094 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1095 host side (use 'qm terminal' to open a terminal connection).
1096
1097 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1098 use with special care.
1099
1100 CAUTION: Experimental! User reported problems with this option.
1101 EODESCR
1102 };
1103
1104 my $paralleldesc= {
1105         optional => 1,
1106         type => 'string',
1107         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1108         description =>  "Map host parallel devices (n is 0 to 2).",
1109         verbose_description =>  <<EODESCR,
1110 Map host parallel devices (n is 0 to 2).
1111
1112 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1113 machines - use with special care.
1114
1115 CAUTION: Experimental! User reported problems with this option.
1116 EODESCR
1117 };
1118
1119 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1120     $confdesc->{"parallel$i"} = $paralleldesc;
1121 }
1122
1123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1124     $confdesc->{"serial$i"} = $serialdesc;
1125 }
1126
1127 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1128     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1129 }
1130
1131 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1132     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1133 }
1134
1135 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1136     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1137 }
1138
1139 my $boot_fmt = {
1140     legacy => {
1141         optional => 1,
1142         default_key => 1,
1143         type => 'string',
1144         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1145                      . " Deprecated, use 'order=' instead.",
1146         pattern => '[acdn]{1,4}',
1147         format_description => "[acdn]{1,4}",
1148
1149         # note: this is also the fallback if boot: is not given at all
1150         default => 'cdn',
1151     },
1152     order => {
1153         optional => 1,
1154         type => 'string',
1155         format => 'pve-qm-bootdev-list',
1156         format_description => "device[;device...]",
1157         description => <<EODESC,
1158 The guest will attempt to boot from devices in the order they appear here.
1159
1160 Disks, optical drives and passed-through storage USB devices will be directly
1161 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1162 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1163
1164 Note that only devices in this list will be marked as bootable and thus loaded
1165 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1166 (e.g. software-raid), you need to specify all of them here.
1167
1168 Overrides the deprecated 'legacy=[acdn]*' value when given.
1169 EODESC
1170     },
1171 };
1172 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1173
1174 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1175 sub verify_bootdev {
1176     my ($dev, $noerr) = @_;
1177
1178     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1179     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1180
1181     my $check = sub {
1182         my ($base) = @_;
1183         return 0 if $dev !~ m/^$base\d+$/;
1184         return 0 if !$confdesc->{$dev};
1185         return 1;
1186     };
1187
1188     return $dev if $check->("net");
1189     return $dev if $check->("usb");
1190     return $dev if $check->("hostpci");
1191
1192     return if $noerr;
1193     die "invalid boot device '$dev'\n";
1194 }
1195
1196 sub print_bootorder {
1197     my ($devs) = @_;
1198     return "" if !@$devs;
1199     my $data = { order => join(';', @$devs) };
1200     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1201 }
1202
1203 my $kvm_api_version = 0;
1204
1205 sub kvm_version {
1206     return $kvm_api_version if $kvm_api_version;
1207
1208     open my $fh, '<', '/dev/kvm' or return;
1209
1210     # 0xae00 => KVM_GET_API_VERSION
1211     $kvm_api_version = ioctl($fh, 0xae00, 0);
1212     close($fh);
1213
1214     return $kvm_api_version;
1215 }
1216
1217 my $kvm_user_version = {};
1218 my $kvm_mtime = {};
1219
1220 sub kvm_user_version {
1221     my ($binary) = @_;
1222
1223     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1224     my $st = stat($binary);
1225
1226     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1227     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1228         $cachedmtime == $st->mtime;
1229
1230     $kvm_user_version->{$binary} = 'unknown';
1231     $kvm_mtime->{$binary} = $st->mtime;
1232
1233     my $code = sub {
1234         my $line = shift;
1235         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1236             $kvm_user_version->{$binary} = $2;
1237         }
1238     };
1239
1240     eval { run_command([$binary, '--version'], outfunc => $code); };
1241     warn $@ if $@;
1242
1243     return $kvm_user_version->{$binary};
1244
1245 }
1246 my sub extract_version {
1247     my ($machine_type, $version) = @_;
1248     $version = kvm_user_version() if !defined($version);
1249     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1250 }
1251
1252 sub kernel_has_vhost_net {
1253     return -c '/dev/vhost-net';
1254 }
1255
1256 sub option_exists {
1257     my $key = shift;
1258     return defined($confdesc->{$key});
1259 }
1260
1261 my $cdrom_path;
1262 sub get_cdrom_path {
1263
1264     return $cdrom_path if defined($cdrom_path);
1265
1266     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1267
1268     if (!defined($cdrom_path)) {
1269         log_warn("no physical CD-ROM available, ignoring");
1270         $cdrom_path = '';
1271     }
1272
1273     return $cdrom_path;
1274 }
1275
1276 sub get_iso_path {
1277     my ($storecfg, $vmid, $cdrom) = @_;
1278
1279     if ($cdrom eq 'cdrom') {
1280         return get_cdrom_path();
1281     } elsif ($cdrom eq 'none') {
1282         return '';
1283     } elsif ($cdrom =~ m|^/|) {
1284         return $cdrom;
1285     } else {
1286         return PVE::Storage::path($storecfg, $cdrom);
1287     }
1288 }
1289
1290 # try to convert old style file names to volume IDs
1291 sub filename_to_volume_id {
1292     my ($vmid, $file, $media) = @_;
1293
1294      if (!($file eq 'none' || $file eq 'cdrom' ||
1295           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1296
1297         return if $file =~ m|/|;
1298
1299         if ($media && $media eq 'cdrom') {
1300             $file = "local:iso/$file";
1301         } else {
1302             $file = "local:$vmid/$file";
1303         }
1304     }
1305
1306     return $file;
1307 }
1308
1309 sub verify_media_type {
1310     my ($opt, $vtype, $media) = @_;
1311
1312     return if !$media;
1313
1314     my $etype;
1315     if ($media eq 'disk') {
1316         $etype = 'images';
1317     } elsif ($media eq 'cdrom') {
1318         $etype = 'iso';
1319     } else {
1320         die "internal error";
1321     }
1322
1323     return if ($vtype eq $etype);
1324
1325     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1326 }
1327
1328 sub cleanup_drive_path {
1329     my ($opt, $storecfg, $drive) = @_;
1330
1331     # try to convert filesystem paths to volume IDs
1332
1333     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1334         ($drive->{file} !~ m|^/dev/.+|) &&
1335         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1336         ($drive->{file} !~ m/^\d+$/)) {
1337         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1338         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1339             if !$vtype;
1340         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1341         verify_media_type($opt, $vtype, $drive->{media});
1342         $drive->{file} = $volid;
1343     }
1344
1345     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1346 }
1347
1348 sub parse_hotplug_features {
1349     my ($data) = @_;
1350
1351     my $res = {};
1352
1353     return $res if $data eq '0';
1354
1355     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1356
1357     foreach my $feature (PVE::Tools::split_list($data)) {
1358         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1359             $res->{$1} = 1;
1360         } else {
1361             die "invalid hotplug feature '$feature'\n";
1362         }
1363     }
1364     return $res;
1365 }
1366
1367 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1368 sub pve_verify_hotplug_features {
1369     my ($value, $noerr) = @_;
1370
1371     return $value if parse_hotplug_features($value);
1372
1373     return if $noerr;
1374
1375     die "unable to parse hotplug option\n";
1376 }
1377
1378 sub scsi_inquiry {
1379     my($fh, $noerr) = @_;
1380
1381     my $SG_IO = 0x2285;
1382     my $SG_GET_VERSION_NUM = 0x2282;
1383
1384     my $versionbuf = "\x00" x 8;
1385     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1386     if (!$ret) {
1387         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1388         return;
1389     }
1390     my $version = unpack("I", $versionbuf);
1391     if ($version < 30000) {
1392         die "scsi generic interface too old\n"  if !$noerr;
1393         return;
1394     }
1395
1396     my $buf = "\x00" x 36;
1397     my $sensebuf = "\x00" x 8;
1398     my $cmd = pack("C x3 C x1", 0x12, 36);
1399
1400     # see /usr/include/scsi/sg.h
1401     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1402
1403     my $packet = pack(
1404         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1405     );
1406
1407     $ret = ioctl($fh, $SG_IO, $packet);
1408     if (!$ret) {
1409         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1410         return;
1411     }
1412
1413     my @res = unpack($sg_io_hdr_t, $packet);
1414     if ($res[17] || $res[18]) {
1415         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1416         return;
1417     }
1418
1419     my $res = {};
1420     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1421
1422     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1423     $res->{type} &= 0x1F;
1424
1425     return $res;
1426 }
1427
1428 sub path_is_scsi {
1429     my ($path) = @_;
1430
1431     my $fh = IO::File->new("+<$path") || return;
1432     my $res = scsi_inquiry($fh, 1);
1433     close($fh);
1434
1435     return $res;
1436 }
1437
1438 sub print_tabletdevice_full {
1439     my ($conf, $arch) = @_;
1440
1441     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1442
1443     # we use uhci for old VMs because tablet driver was buggy in older qemu
1444     my $usbbus;
1445     if ($q35 || $arch eq 'aarch64') {
1446         $usbbus = 'ehci';
1447     } else {
1448         $usbbus = 'uhci';
1449     }
1450
1451     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1452 }
1453
1454 sub print_keyboarddevice_full {
1455     my ($conf, $arch) = @_;
1456
1457     return if $arch ne 'aarch64';
1458
1459     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1460 }
1461
1462 my sub get_drive_id {
1463     my ($drive) = @_;
1464     return "$drive->{interface}$drive->{index}";
1465 }
1466
1467 sub print_drivedevice_full {
1468     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1469
1470     my $device = '';
1471     my $maxdev = 0;
1472
1473     my $drive_id = get_drive_id($drive);
1474     if ($drive->{interface} eq 'virtio') {
1475         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1476         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1477         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1478     } elsif ($drive->{interface} eq 'scsi') {
1479
1480         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1481         my $unit = $drive->{index} % $maxdev;
1482         my $devicetype = 'hd';
1483         my $path = '';
1484         if (drive_is_cdrom($drive)) {
1485             $devicetype = 'cd';
1486         } else {
1487             if ($drive->{file} =~ m|^/|) {
1488                 $path = $drive->{file};
1489                 if (my $info = path_is_scsi($path)) {
1490                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1491                         $devicetype = 'block';
1492                     } elsif ($info->{type} == 1) { # tape
1493                         $devicetype = 'generic';
1494                     }
1495                 }
1496             } else {
1497                  $path = PVE::Storage::path($storecfg, $drive->{file});
1498             }
1499
1500             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1501             my $version = extract_version($machine_type, kvm_user_version());
1502             if ($path =~ m/^iscsi\:\/\// &&
1503                !min_version($version, 4, 1)) {
1504                 $devicetype = 'generic';
1505             }
1506         }
1507
1508         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1509             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1510         } else {
1511             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1512                 .",lun=$drive->{index}";
1513         }
1514         $device .= ",drive=drive-$drive_id,id=$drive_id";
1515
1516         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1517             $device .= ",rotation_rate=1";
1518         }
1519         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1520
1521     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1522         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1523         my $controller = int($drive->{index} / $maxdev);
1524         my $unit = $drive->{index} % $maxdev;
1525
1526         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1527         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1528         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1529         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1530         # were before. Move odd ones up by 2 where they don't clash.
1531         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1532             $controller += 2 * ($unit % 2);
1533             $unit = 0;
1534         }
1535
1536         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1537
1538         $device = "ide-$devicetype";
1539         if ($drive->{interface} eq 'ide') {
1540             $device .= ",bus=ide.$controller,unit=$unit";
1541         } else {
1542             $device .= ",bus=ahci$controller.$unit";
1543         }
1544         $device .= ",drive=drive-$drive_id,id=$drive_id";
1545
1546         if ($devicetype eq 'hd') {
1547             if (my $model = $drive->{model}) {
1548                 $model = URI::Escape::uri_unescape($model);
1549                 $device .= ",model=$model";
1550             }
1551             if ($drive->{ssd}) {
1552                 $device .= ",rotation_rate=1";
1553             }
1554         }
1555         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1556     } elsif ($drive->{interface} eq 'usb') {
1557         die "implement me";
1558         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1559     } else {
1560         die "unsupported interface type";
1561     }
1562
1563     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1564
1565     if (my $serial = $drive->{serial}) {
1566         $serial = URI::Escape::uri_unescape($serial);
1567         $device .= ",serial=$serial";
1568     }
1569
1570
1571     return $device;
1572 }
1573
1574 sub get_initiator_name {
1575     my $initiator;
1576
1577     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1578     while (defined(my $line = <$fh>)) {
1579         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1580         $initiator = $1;
1581         last;
1582     }
1583     $fh->close();
1584
1585     return $initiator;
1586 }
1587
1588 my sub storage_allows_io_uring_default {
1589     my ($scfg, $cache_direct) = @_;
1590
1591     # io_uring with cache mode writeback or writethrough on krbd will hang...
1592     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1593
1594     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1595     # sometimes, just plain disable...
1596     return if $scfg && $scfg->{type} eq 'lvm';
1597
1598     # io_uring causes problems when used with CIFS since kernel 5.15
1599     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1600     return if $scfg && $scfg->{type} eq 'cifs';
1601
1602     return 1;
1603 }
1604
1605 my sub drive_uses_cache_direct {
1606     my ($drive, $scfg) = @_;
1607
1608     my $cache_direct = 0;
1609
1610     if (my $cache = $drive->{cache}) {
1611         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1612     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1613         $cache_direct = 1;
1614     }
1615
1616     return $cache_direct;
1617 }
1618
1619 sub print_drive_commandline_full {
1620     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1621
1622     my $path;
1623     my $volid = $drive->{file};
1624     my $format = $drive->{format};
1625     my $drive_id = get_drive_id($drive);
1626
1627     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1628     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1629
1630     if (drive_is_cdrom($drive)) {
1631         $path = get_iso_path($storecfg, $vmid, $volid);
1632         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1633     } else {
1634         if ($storeid) {
1635             $path = PVE::Storage::path($storecfg, $volid);
1636             $format //= qemu_img_format($scfg, $volname);
1637         } else {
1638             $path = $volid;
1639             $format //= "raw";
1640         }
1641    }
1642
1643    my $is_rbd = $path =~ m/^rbd:/;
1644
1645     my $opts = '';
1646     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1647     foreach my $o (@qemu_drive_options) {
1648         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1649     }
1650
1651     # snapshot only accepts on|off
1652     if (defined($drive->{snapshot})) {
1653         my $v = $drive->{snapshot} ? 'on' : 'off';
1654         $opts .= ",snapshot=$v";
1655     }
1656
1657     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1658         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1659     }
1660
1661     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1662         my ($dir, $qmpname) = @$type;
1663         if (my $v = $drive->{"mbps$dir"}) {
1664             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1665         }
1666         if (my $v = $drive->{"mbps${dir}_max"}) {
1667             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1668         }
1669         if (my $v = $drive->{"bps${dir}_max_length"}) {
1670             $opts .= ",throttling.bps$qmpname-max-length=$v";
1671         }
1672         if (my $v = $drive->{"iops${dir}"}) {
1673             $opts .= ",throttling.iops$qmpname=$v";
1674         }
1675         if (my $v = $drive->{"iops${dir}_max"}) {
1676             $opts .= ",throttling.iops$qmpname-max=$v";
1677         }
1678         if (my $v = $drive->{"iops${dir}_max_length"}) {
1679             $opts .= ",throttling.iops$qmpname-max-length=$v";
1680         }
1681     }
1682
1683     if ($pbs_name) {
1684         $format = "rbd" if $is_rbd;
1685         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1686             if !$format;
1687         $opts .= ",format=alloc-track,file.driver=$format";
1688     } elsif ($format) {
1689         $opts .= ",format=$format";
1690     }
1691
1692     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1693
1694     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1695
1696     if (!$drive->{aio}) {
1697         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1698             # io_uring supports all cache modes
1699             $opts .= ",aio=io_uring";
1700         } else {
1701             # aio native works only with O_DIRECT
1702             if($cache_direct) {
1703                 $opts .= ",aio=native";
1704             } else {
1705                 $opts .= ",aio=threads";
1706             }
1707         }
1708     }
1709
1710     if (!drive_is_cdrom($drive)) {
1711         my $detectzeroes;
1712         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1713             $detectzeroes = 'off';
1714         } elsif ($drive->{discard}) {
1715             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1716         } else {
1717             # This used to be our default with discard not being specified:
1718             $detectzeroes = 'on';
1719         }
1720
1721         # note: 'detect-zeroes' works per blockdev and we want it to persist
1722         # after the alloc-track is removed, so put it on 'file' directly
1723         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1724         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1725     }
1726
1727     if ($pbs_name) {
1728         $opts .= ",backing=$pbs_name";
1729         $opts .= ",auto-remove=on";
1730     }
1731
1732     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1733     my $file_param = "file";
1734     if ($pbs_name) {
1735         # non-rbd drivers require the underlying file to be a seperate block
1736         # node, so add a second .file indirection
1737         $file_param .= ".file" if !$is_rbd;
1738         $file_param .= ".filename";
1739     }
1740     my $pathinfo = $path ? "$file_param=$path," : '';
1741
1742     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1743 }
1744
1745 sub print_pbs_blockdev {
1746     my ($pbs_conf, $pbs_name) = @_;
1747     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1748     $blockdev .= ",repository=$pbs_conf->{repository}";
1749     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1750     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1751     $blockdev .= ",archive=$pbs_conf->{archive}";
1752     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1753     return $blockdev;
1754 }
1755
1756 sub print_netdevice_full {
1757     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1758
1759     my $device = $net->{model};
1760     if ($net->{model} eq 'virtio') {
1761          $device = 'virtio-net-pci';
1762      };
1763
1764     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1765     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1766     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1767         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1768         # and out of each queue plus one config interrupt and control vector queue
1769         my $vectors = $net->{queues} * 2 + 2;
1770         $tmpstr .= ",vectors=$vectors,mq=on";
1771         if (min_version($machine_version, 7, 1)) {
1772             $tmpstr .= ",packed=on";
1773         }
1774     }
1775
1776     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1777         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1778     }
1779
1780     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1781
1782     if (my $mtu = $net->{mtu}) {
1783         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1784             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1785             if ($mtu == 1) {
1786                  $mtu = $bridge_mtu;
1787             } elsif ($mtu < 576) {
1788                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1789             } elsif ($mtu > $bridge_mtu) {
1790                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1791             }
1792             $tmpstr .= ",host_mtu=$mtu";
1793         } else {
1794             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1795         }
1796     }
1797
1798     if ($use_old_bios_files) {
1799         my $romfile;
1800         if ($device eq 'virtio-net-pci') {
1801             $romfile = 'pxe-virtio.rom';
1802         } elsif ($device eq 'e1000') {
1803             $romfile = 'pxe-e1000.rom';
1804         } elsif ($device eq 'e1000e') {
1805             $romfile = 'pxe-e1000e.rom';
1806         } elsif ($device eq 'ne2k') {
1807             $romfile = 'pxe-ne2k_pci.rom';
1808         } elsif ($device eq 'pcnet') {
1809             $romfile = 'pxe-pcnet.rom';
1810         } elsif ($device eq 'rtl8139') {
1811             $romfile = 'pxe-rtl8139.rom';
1812         }
1813         $tmpstr .= ",romfile=$romfile" if $romfile;
1814     }
1815
1816     return $tmpstr;
1817 }
1818
1819 sub print_netdev_full {
1820     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1821
1822     my $i = '';
1823     if ($netid =~ m/^net(\d+)$/) {
1824         $i = int($1);
1825     }
1826
1827     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1828
1829     my $ifname = "tap${vmid}i$i";
1830
1831     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1832     die "interface name '$ifname' is too long (max 15 character)\n"
1833         if length($ifname) >= 16;
1834
1835     my $vhostparam = '';
1836     if (is_native($arch)) {
1837         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1838     }
1839
1840     my $vmname = $conf->{name} || "vm$vmid";
1841
1842     my $netdev = "";
1843     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1844
1845     if ($net->{bridge}) {
1846         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1847             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1848     } else {
1849         $netdev = "type=user,id=$netid,hostname=$vmname";
1850     }
1851
1852     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1853
1854     return $netdev;
1855 }
1856
1857 my $vga_map = {
1858     'cirrus' => 'cirrus-vga',
1859     'std' => 'VGA',
1860     'vmware' => 'vmware-svga',
1861     'virtio' => 'virtio-vga',
1862     'virtio-gl' => 'virtio-vga-gl',
1863 };
1864
1865 sub print_vga_device {
1866     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1867
1868     my $type = $vga_map->{$vga->{type}};
1869     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1870         $type = 'virtio-gpu';
1871     }
1872     my $vgamem_mb = $vga->{memory};
1873
1874     my $max_outputs = '';
1875     if ($qxlnum) {
1876         $type = $id ? 'qxl' : 'qxl-vga';
1877
1878         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1879             # set max outputs so linux can have up to 4 qxl displays with one device
1880             if (min_version($machine_version, 4, 1)) {
1881                 $max_outputs = ",max_outputs=4";
1882             }
1883         }
1884     }
1885
1886     die "no devicetype for $vga->{type}\n" if !$type;
1887
1888     my $memory = "";
1889     if ($vgamem_mb) {
1890         if ($vga->{type} =~ /^virtio/) {
1891             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1892             $memory = ",max_hostmem=$bytes";
1893         } elsif ($qxlnum) {
1894             # from https://www.spice-space.org/multiple-monitors.html
1895             $memory = ",vgamem_mb=$vga->{memory}";
1896             my $ram = $vgamem_mb * 4;
1897             my $vram = $vgamem_mb * 2;
1898             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1899         } else {
1900             $memory = ",vgamem_mb=$vga->{memory}";
1901         }
1902     } elsif ($qxlnum && $id) {
1903         $memory = ",ram_size=67108864,vram_size=33554432";
1904     }
1905
1906     my $edidoff = "";
1907     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1908         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1909     }
1910
1911     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1912     my $vgaid = "vga" . ($id // '');
1913     my $pciaddr;
1914     if ($q35 && $vgaid eq 'vga') {
1915         # the first display uses pcie.0 bus on q35 machines
1916         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1917     } else {
1918         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1919     }
1920
1921     if ($vga->{type} eq 'virtio-gl') {
1922         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1923         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1924             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1925
1926         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1927             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1928     }
1929
1930     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1931 }
1932
1933 sub parse_number_sets {
1934     my ($set) = @_;
1935     my $res = [];
1936     foreach my $part (split(/;/, $set)) {
1937         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1938             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1939             push @$res, [ $1, $2 ];
1940         } else {
1941             die "invalid range: $part\n";
1942         }
1943     }
1944     return $res;
1945 }
1946
1947 sub parse_numa {
1948     my ($data) = @_;
1949
1950     my $res = parse_property_string($numa_fmt, $data);
1951     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1952     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1953     return $res;
1954 }
1955
1956 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1957 sub parse_net {
1958     my ($data, $disable_mac_autogen) = @_;
1959
1960     my $res = eval { parse_property_string($net_fmt, $data) };
1961     if ($@) {
1962         warn $@;
1963         return;
1964     }
1965     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1966         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1967         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1968     }
1969     return $res;
1970 }
1971
1972 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1973 sub parse_ipconfig {
1974     my ($data) = @_;
1975
1976     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1977     if ($@) {
1978         warn $@;
1979         return;
1980     }
1981
1982     if ($res->{gw} && !$res->{ip}) {
1983         warn 'gateway specified without specifying an IP address';
1984         return;
1985     }
1986     if ($res->{gw6} && !$res->{ip6}) {
1987         warn 'IPv6 gateway specified without specifying an IPv6 address';
1988         return;
1989     }
1990     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1991         warn 'gateway specified together with DHCP';
1992         return;
1993     }
1994     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1995         # gw6 + auto/dhcp
1996         warn "IPv6 gateway specified together with $res->{ip6} address";
1997         return;
1998     }
1999
2000     if (!$res->{ip} && !$res->{ip6}) {
2001         return { ip => 'dhcp', ip6 => 'dhcp' };
2002     }
2003
2004     return $res;
2005 }
2006
2007 sub print_net {
2008     my $net = shift;
2009
2010     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2011 }
2012
2013 sub add_random_macs {
2014     my ($settings) = @_;
2015
2016     foreach my $opt (keys %$settings) {
2017         next if $opt !~ m/^net(\d+)$/;
2018         my $net = parse_net($settings->{$opt});
2019         next if !$net;
2020         $settings->{$opt} = print_net($net);
2021     }
2022 }
2023
2024 sub vm_is_volid_owner {
2025     my ($storecfg, $vmid, $volid) = @_;
2026
2027     if ($volid !~  m|^/|) {
2028         my ($path, $owner);
2029         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2030         if ($owner && ($owner == $vmid)) {
2031             return 1;
2032         }
2033     }
2034
2035     return;
2036 }
2037
2038 sub vmconfig_register_unused_drive {
2039     my ($storecfg, $vmid, $conf, $drive) = @_;
2040
2041     if (drive_is_cloudinit($drive)) {
2042         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2043         warn $@ if $@;
2044         delete $conf->{cloudinit};
2045     } elsif (!drive_is_cdrom($drive)) {
2046         my $volid = $drive->{file};
2047         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2048             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2049         }
2050     }
2051 }
2052
2053 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2054 my $smbios1_fmt = {
2055     uuid => {
2056         type => 'string',
2057         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2058         format_description => 'UUID',
2059         description => "Set SMBIOS1 UUID.",
2060         optional => 1,
2061     },
2062     version => {
2063         type => 'string',
2064         pattern => '[A-Za-z0-9+\/]+={0,2}',
2065         format_description => 'Base64 encoded string',
2066         description => "Set SMBIOS1 version.",
2067         optional => 1,
2068     },
2069     serial => {
2070         type => 'string',
2071         pattern => '[A-Za-z0-9+\/]+={0,2}',
2072         format_description => 'Base64 encoded string',
2073         description => "Set SMBIOS1 serial number.",
2074         optional => 1,
2075     },
2076     manufacturer => {
2077         type => 'string',
2078         pattern => '[A-Za-z0-9+\/]+={0,2}',
2079         format_description => 'Base64 encoded string',
2080         description => "Set SMBIOS1 manufacturer.",
2081         optional => 1,
2082     },
2083     product => {
2084         type => 'string',
2085         pattern => '[A-Za-z0-9+\/]+={0,2}',
2086         format_description => 'Base64 encoded string',
2087         description => "Set SMBIOS1 product ID.",
2088         optional => 1,
2089     },
2090     sku => {
2091         type => 'string',
2092         pattern => '[A-Za-z0-9+\/]+={0,2}',
2093         format_description => 'Base64 encoded string',
2094         description => "Set SMBIOS1 SKU string.",
2095         optional => 1,
2096     },
2097     family => {
2098         type => 'string',
2099         pattern => '[A-Za-z0-9+\/]+={0,2}',
2100         format_description => 'Base64 encoded string',
2101         description => "Set SMBIOS1 family string.",
2102         optional => 1,
2103     },
2104     base64 => {
2105         type => 'boolean',
2106         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2107         optional => 1,
2108     },
2109 };
2110
2111 sub parse_smbios1 {
2112     my ($data) = @_;
2113
2114     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2115     warn $@ if $@;
2116     return $res;
2117 }
2118
2119 sub print_smbios1 {
2120     my ($smbios1) = @_;
2121     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2122 }
2123
2124 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2125
2126 sub parse_watchdog {
2127     my ($value) = @_;
2128
2129     return if !$value;
2130
2131     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2132     warn $@ if $@;
2133     return $res;
2134 }
2135
2136 sub parse_guest_agent {
2137     my ($conf) = @_;
2138
2139     return {} if !defined($conf->{agent});
2140
2141     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2142     warn $@ if $@;
2143
2144     # if the agent is disabled ignore the other potentially set properties
2145     return {} if !$res->{enabled};
2146     return $res;
2147 }
2148
2149 sub get_qga_key {
2150     my ($conf, $key) = @_;
2151     return undef if !defined($conf->{agent});
2152
2153     my $agent = parse_guest_agent($conf);
2154     return $agent->{$key};
2155 }
2156
2157 sub parse_vga {
2158     my ($value) = @_;
2159
2160     return {} if !$value;
2161     my $res = eval { parse_property_string($vga_fmt, $value) };
2162     warn $@ if $@;
2163     return $res;
2164 }
2165
2166 sub parse_rng {
2167     my ($value) = @_;
2168
2169     return if !$value;
2170
2171     my $res = eval { parse_property_string($rng_fmt, $value) };
2172     warn $@ if $@;
2173     return $res;
2174 }
2175
2176 sub parse_meta_info {
2177     my ($value) = @_;
2178
2179     return if !$value;
2180
2181     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2182     warn $@ if $@;
2183     return $res;
2184 }
2185
2186 sub new_meta_info_string {
2187     my () = @_; # for now do not allow to override any value
2188
2189     return PVE::JSONSchema::print_property_string(
2190         {
2191             'creation-qemu' => kvm_user_version(),
2192             ctime => "". int(time()),
2193         },
2194         $meta_info_fmt
2195     );
2196 }
2197
2198 sub qemu_created_version_fixups {
2199     my ($conf, $forcemachine, $kvmver) = @_;
2200
2201     my $meta = parse_meta_info($conf->{meta}) // {};
2202     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2203
2204     # check if we need to apply some handling for VMs that always use the latest machine version but
2205     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2206     # would be required (we do not want to pin machine version for non-windows OS type)
2207     if (
2208         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2209         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2210         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2211         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2212     ) {
2213         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2214         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2215             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2216             # and thus with the predictable interface naming of systemd
2217             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2218         }
2219     }
2220     return;
2221 }
2222
2223 # add JSON properties for create and set function
2224 sub json_config_properties {
2225     my ($prop, $with_disk_alloc) = @_;
2226
2227     my $skip_json_config_opts = {
2228         parent => 1,
2229         snaptime => 1,
2230         vmstate => 1,
2231         runningmachine => 1,
2232         runningcpu => 1,
2233         meta => 1,
2234     };
2235
2236     foreach my $opt (keys %$confdesc) {
2237         next if $skip_json_config_opts->{$opt};
2238
2239         if ($with_disk_alloc && is_valid_drivename($opt)) {
2240             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2241         } else {
2242             $prop->{$opt} = $confdesc->{$opt};
2243         }
2244     }
2245
2246     return $prop;
2247 }
2248
2249 # Properties that we can read from an OVF file
2250 sub json_ovf_properties {
2251     my $prop = {};
2252
2253     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2254         $prop->{$device} = {
2255             type => 'string',
2256             format => 'pve-volume-id-or-absolute-path',
2257             description => "Disk image that gets imported to $device",
2258             optional => 1,
2259         };
2260     }
2261
2262     $prop->{cores} = {
2263         type => 'integer',
2264         description => "The number of CPU cores.",
2265         optional => 1,
2266     };
2267     $prop->{memory} = {
2268         type => 'integer',
2269         description => "Amount of RAM for the VM in MB.",
2270         optional => 1,
2271     };
2272     $prop->{name} = {
2273         type => 'string',
2274         description => "Name of the VM.",
2275         optional => 1,
2276     };
2277
2278     return $prop;
2279 }
2280
2281 # return copy of $confdesc_cloudinit to generate documentation
2282 sub cloudinit_config_properties {
2283
2284     return dclone($confdesc_cloudinit);
2285 }
2286
2287 sub cloudinit_pending_properties {
2288     my $p = {
2289         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2290         name => 1,
2291     };
2292     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2293     return $p;
2294 }
2295
2296 sub check_type {
2297     my ($key, $value) = @_;
2298
2299     die "unknown setting '$key'\n" if !$confdesc->{$key};
2300
2301     my $type = $confdesc->{$key}->{type};
2302
2303     if (!defined($value)) {
2304         die "got undefined value\n";
2305     }
2306
2307     if ($value =~ m/[\n\r]/) {
2308         die "property contains a line feed\n";
2309     }
2310
2311     if ($type eq 'boolean') {
2312         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2313         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2314         die "type check ('boolean') failed - got '$value'\n";
2315     } elsif ($type eq 'integer') {
2316         return int($1) if $value =~ m/^(\d+)$/;
2317         die "type check ('integer') failed - got '$value'\n";
2318     } elsif ($type eq 'number') {
2319         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2320         die "type check ('number') failed - got '$value'\n";
2321     } elsif ($type eq 'string') {
2322         if (my $fmt = $confdesc->{$key}->{format}) {
2323             PVE::JSONSchema::check_format($fmt, $value);
2324             return $value;
2325         }
2326         $value =~ s/^\"(.*)\"$/$1/;
2327         return $value;
2328     } else {
2329         die "internal error"
2330     }
2331 }
2332
2333 sub destroy_vm {
2334     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2335
2336     my $conf = PVE::QemuConfig->load_config($vmid);
2337
2338     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2339         PVE::QemuConfig->check_lock($conf);
2340     }
2341
2342     if ($conf->{template}) {
2343         # check if any base image is still used by a linked clone
2344         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2345                 my ($ds, $drive) = @_;
2346                 return if drive_is_cdrom($drive);
2347
2348                 my $volid = $drive->{file};
2349                 return if !$volid || $volid =~ m|^/|;
2350
2351                 die "base volume '$volid' is still in use by linked cloned\n"
2352                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2353
2354         });
2355     }
2356
2357     my $volids = {};
2358     my $remove_owned_drive = sub {
2359         my ($ds, $drive) = @_;
2360         return if drive_is_cdrom($drive, 1);
2361
2362         my $volid = $drive->{file};
2363         return if !$volid || $volid =~ m|^/|;
2364         return if $volids->{$volid};
2365
2366         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2367         return if !$path || !$owner || ($owner != $vmid);
2368
2369         $volids->{$volid} = 1;
2370         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2371         warn "Could not remove disk '$volid', check manually: $@" if $@;
2372     };
2373
2374     # only remove disks owned by this VM (referenced in the config)
2375     my $include_opts = {
2376         include_unused => 1,
2377         extra_keys => ['vmstate'],
2378     };
2379     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2380
2381     for my $snap (values %{$conf->{snapshots}}) {
2382         next if !defined($snap->{vmstate});
2383         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2384         next if !defined($drive);
2385         $remove_owned_drive->('vmstate', $drive);
2386     }
2387
2388     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2389
2390     if ($purge_unreferenced) { # also remove unreferenced disk
2391         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2392         PVE::Storage::foreach_volid($vmdisks, sub {
2393             my ($volid, $sid, $volname, $d) = @_;
2394             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2395             warn $@ if $@;
2396         });
2397     }
2398
2399     if (defined $replacement_conf) {
2400         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2401     } else {
2402         PVE::QemuConfig->destroy_config($vmid);
2403     }
2404 }
2405
2406 sub parse_vm_config {
2407     my ($filename, $raw, $strict) = @_;
2408
2409     return if !defined($raw);
2410
2411     my $res = {
2412         digest => Digest::SHA::sha1_hex($raw),
2413         snapshots => {},
2414         pending => {},
2415         cloudinit => {},
2416     };
2417
2418     my $handle_error = sub {
2419         my ($msg) = @_;
2420
2421         if ($strict) {
2422             die $msg;
2423         } else {
2424             warn $msg;
2425         }
2426     };
2427
2428     $filename =~ m|/qemu-server/(\d+)\.conf$|
2429         || die "got strange filename '$filename'";
2430
2431     my $vmid = $1;
2432
2433     my $conf = $res;
2434     my $descr;
2435     my $finish_description = sub {
2436         if (defined($descr)) {
2437             $descr =~ s/\s+$//;
2438             $conf->{description} = $descr;
2439         }
2440         $descr = undef;
2441     };
2442     my $section = '';
2443
2444     my @lines = split(/\n/, $raw);
2445     foreach my $line (@lines) {
2446         next if $line =~ m/^\s*$/;
2447
2448         if ($line =~ m/^\[PENDING\]\s*$/i) {
2449             $section = 'pending';
2450             $finish_description->();
2451             $conf = $res->{$section} = {};
2452             next;
2453         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2454             $section = 'cloudinit';
2455             $finish_description->();
2456             $conf = $res->{$section} = {};
2457             next;
2458
2459         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2460             $section = $1;
2461             $finish_description->();
2462             $conf = $res->{snapshots}->{$section} = {};
2463             next;
2464         }
2465
2466         if ($line =~ m/^\#(.*)$/) {
2467             $descr = '' if !defined($descr);
2468             $descr .= PVE::Tools::decode_text($1) . "\n";
2469             next;
2470         }
2471
2472         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2473             $descr = '' if !defined($descr);
2474             $descr .= PVE::Tools::decode_text($2);
2475         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2476             $conf->{snapstate} = $1;
2477         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2478             my $key = $1;
2479             my $value = $2;
2480             $conf->{$key} = $value;
2481         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2482             my $value = $1;
2483             if ($section eq 'pending') {
2484                 $conf->{delete} = $value; # we parse this later
2485             } else {
2486                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2487             }
2488         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2489             my $key = $1;
2490             my $value = $2;
2491             if ($section eq 'cloudinit') {
2492                 # ignore validation only used for informative purpose
2493                 $conf->{$key} = $value;
2494                 next;
2495             }
2496             eval { $value = check_type($key, $value); };
2497             if ($@) {
2498                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2499             } else {
2500                 $key = 'ide2' if $key eq 'cdrom';
2501                 my $fmt = $confdesc->{$key}->{format};
2502                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2503                     my $v = parse_drive($key, $value);
2504                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2505                         $v->{file} = $volid;
2506                         $value = print_drive($v);
2507                     } else {
2508                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2509                         next;
2510                     }
2511                 }
2512
2513                 $conf->{$key} = $value;
2514             }
2515         } else {
2516             $handle_error->("vm $vmid - unable to parse config: $line\n");
2517         }
2518     }
2519
2520     $finish_description->();
2521     delete $res->{snapstate}; # just to be sure
2522
2523     return $res;
2524 }
2525
2526 sub write_vm_config {
2527     my ($filename, $conf) = @_;
2528
2529     delete $conf->{snapstate}; # just to be sure
2530
2531     if ($conf->{cdrom}) {
2532         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2533         $conf->{ide2} = $conf->{cdrom};
2534         delete $conf->{cdrom};
2535     }
2536
2537     # we do not use 'smp' any longer
2538     if ($conf->{sockets}) {
2539         delete $conf->{smp};
2540     } elsif ($conf->{smp}) {
2541         $conf->{sockets} = $conf->{smp};
2542         delete $conf->{cores};
2543         delete $conf->{smp};
2544     }
2545
2546     my $used_volids = {};
2547
2548     my $cleanup_config = sub {
2549         my ($cref, $pending, $snapname) = @_;
2550
2551         foreach my $key (keys %$cref) {
2552             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2553                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2554             my $value = $cref->{$key};
2555             if ($key eq 'delete') {
2556                 die "propertry 'delete' is only allowed in [PENDING]\n"
2557                     if !$pending;
2558                 # fixme: check syntax?
2559                 next;
2560             }
2561             eval { $value = check_type($key, $value); };
2562             die "unable to parse value of '$key' - $@" if $@;
2563
2564             $cref->{$key} = $value;
2565
2566             if (!$snapname && is_valid_drivename($key)) {
2567                 my $drive = parse_drive($key, $value);
2568                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2569             }
2570         }
2571     };
2572
2573     &$cleanup_config($conf);
2574
2575     &$cleanup_config($conf->{pending}, 1);
2576
2577     foreach my $snapname (keys %{$conf->{snapshots}}) {
2578         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2579         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2580     }
2581
2582     # remove 'unusedX' settings if we re-add a volume
2583     foreach my $key (keys %$conf) {
2584         my $value = $conf->{$key};
2585         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2586             delete $conf->{$key};
2587         }
2588     }
2589
2590     my $generate_raw_config = sub {
2591         my ($conf, $pending) = @_;
2592
2593         my $raw = '';
2594
2595         # add description as comment to top of file
2596         if (defined(my $descr = $conf->{description})) {
2597             if ($descr) {
2598                 foreach my $cl (split(/\n/, $descr)) {
2599                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2600                 }
2601             } else {
2602                 $raw .= "#\n" if $pending;
2603             }
2604         }
2605
2606         foreach my $key (sort keys %$conf) {
2607             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2608             $raw .= "$key: $conf->{$key}\n";
2609         }
2610         return $raw;
2611     };
2612
2613     my $raw = &$generate_raw_config($conf);
2614
2615     if (scalar(keys %{$conf->{pending}})){
2616         $raw .= "\n[PENDING]\n";
2617         $raw .= &$generate_raw_config($conf->{pending}, 1);
2618     }
2619
2620     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2621         $raw .= "\n[special:cloudinit]\n";
2622         $raw .= &$generate_raw_config($conf->{cloudinit});
2623     }
2624
2625     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2626         $raw .= "\n[$snapname]\n";
2627         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2628     }
2629
2630     return $raw;
2631 }
2632
2633 sub load_defaults {
2634
2635     my $res = {};
2636
2637     # we use static defaults from our JSON schema configuration
2638     foreach my $key (keys %$confdesc) {
2639         if (defined(my $default = $confdesc->{$key}->{default})) {
2640             $res->{$key} = $default;
2641         }
2642     }
2643
2644     return $res;
2645 }
2646
2647 sub config_list {
2648     my $vmlist = PVE::Cluster::get_vmlist();
2649     my $res = {};
2650     return $res if !$vmlist || !$vmlist->{ids};
2651     my $ids = $vmlist->{ids};
2652     my $nodename = nodename();
2653
2654     foreach my $vmid (keys %$ids) {
2655         my $d = $ids->{$vmid};
2656         next if !$d->{node} || $d->{node} ne $nodename;
2657         next if !$d->{type} || $d->{type} ne 'qemu';
2658         $res->{$vmid}->{exists} = 1;
2659     }
2660     return $res;
2661 }
2662
2663 # test if VM uses local resources (to prevent migration)
2664 sub check_local_resources {
2665     my ($conf, $noerr) = @_;
2666
2667     my @loc_res = ();
2668     my $mapped_res = [];
2669
2670     my $nodelist = PVE::Cluster::get_nodelist();
2671     my $pci_map = PVE::Mapping::PCI::config();
2672     my $usb_map = PVE::Mapping::USB::config();
2673
2674     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2675
2676     my $add_missing_mapping = sub {
2677         my ($type, $key, $id) = @_;
2678         for my $node (@$nodelist) {
2679             my $entry;
2680             if ($type eq 'pci') {
2681                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2682             } elsif ($type eq 'usb') {
2683                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2684             }
2685             if (!scalar($entry->@*)) {
2686                 push @{$missing_mappings_by_node->{$node}}, $key;
2687             }
2688         }
2689     };
2690
2691     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2692     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2693
2694     push @loc_res, "ivshmem" if $conf->{ivshmem};
2695
2696     foreach my $k (keys %$conf) {
2697         if ($k =~ m/^usb/) {
2698             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2699             next if $entry->{host} =~ m/^spice$/i;
2700             if ($entry->{mapping}) {
2701                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2702                 push @$mapped_res, $k;
2703             }
2704         }
2705         if ($k =~ m/^hostpci/) {
2706             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2707             if ($entry->{mapping}) {
2708                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2709                 push @$mapped_res, $k;
2710             }
2711         }
2712         # sockets are safe: they will recreated be on the target side post-migrate
2713         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2714         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2715     }
2716
2717     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2718
2719     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2720 }
2721
2722 # check if used storages are available on all nodes (use by migrate)
2723 sub check_storage_availability {
2724     my ($storecfg, $conf, $node) = @_;
2725
2726     PVE::QemuConfig->foreach_volume($conf, sub {
2727         my ($ds, $drive) = @_;
2728
2729         my $volid = $drive->{file};
2730         return if !$volid;
2731
2732         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2733         return if !$sid;
2734
2735         # check if storage is available on both nodes
2736         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2737         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2738
2739         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2740
2741         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2742             if !$scfg->{content}->{$vtype};
2743    });
2744 }
2745
2746 # list nodes where all VM images are available (used by has_feature API)
2747 sub shared_nodes {
2748     my ($conf, $storecfg) = @_;
2749
2750     my $nodelist = PVE::Cluster::get_nodelist();
2751     my $nodehash = { map { $_ => 1 } @$nodelist };
2752     my $nodename = nodename();
2753
2754     PVE::QemuConfig->foreach_volume($conf, sub {
2755         my ($ds, $drive) = @_;
2756
2757         my $volid = $drive->{file};
2758         return if !$volid;
2759
2760         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2761         if ($storeid) {
2762             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2763             if ($scfg->{disable}) {
2764                 $nodehash = {};
2765             } elsif (my $avail = $scfg->{nodes}) {
2766                 foreach my $node (keys %$nodehash) {
2767                     delete $nodehash->{$node} if !$avail->{$node};
2768                 }
2769             } elsif (!$scfg->{shared}) {
2770                 foreach my $node (keys %$nodehash) {
2771                     delete $nodehash->{$node} if $node ne $nodename
2772                 }
2773             }
2774         }
2775     });
2776
2777     return $nodehash
2778 }
2779
2780 sub check_local_storage_availability {
2781     my ($conf, $storecfg) = @_;
2782
2783     my $nodelist = PVE::Cluster::get_nodelist();
2784     my $nodehash = { map { $_ => {} } @$nodelist };
2785
2786     PVE::QemuConfig->foreach_volume($conf, sub {
2787         my ($ds, $drive) = @_;
2788
2789         my $volid = $drive->{file};
2790         return if !$volid;
2791
2792         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2793         if ($storeid) {
2794             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2795
2796             if ($scfg->{disable}) {
2797                 foreach my $node (keys %$nodehash) {
2798                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2799                 }
2800             } elsif (my $avail = $scfg->{nodes}) {
2801                 foreach my $node (keys %$nodehash) {
2802                     if (!$avail->{$node}) {
2803                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2804                     }
2805                 }
2806             }
2807         }
2808     });
2809
2810     foreach my $node (values %$nodehash) {
2811         if (my $unavail = $node->{unavailable_storages}) {
2812             $node->{unavailable_storages} = [ sort keys %$unavail ];
2813         }
2814     }
2815
2816     return $nodehash
2817 }
2818
2819 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2820 sub check_running {
2821     my ($vmid, $nocheck, $node) = @_;
2822
2823     # $nocheck is set when called during a migration, in which case the config
2824     # file might still or already reside on the *other* node
2825     # - because rename has already happened, and current node is source
2826     # - because rename hasn't happened yet, and current node is target
2827     # - because rename has happened, current node is target, but hasn't yet
2828     # processed it yet
2829     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2830     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2831 }
2832
2833 sub vzlist {
2834
2835     my $vzlist = config_list();
2836
2837     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2838
2839     while (defined(my $de = $fd->read)) {
2840         next if $de !~ m/^(\d+)\.pid$/;
2841         my $vmid = $1;
2842         next if !defined($vzlist->{$vmid});
2843         if (my $pid = check_running($vmid)) {
2844             $vzlist->{$vmid}->{pid} = $pid;
2845         }
2846     }
2847
2848     return $vzlist;
2849 }
2850
2851 our $vmstatus_return_properties = {
2852     vmid => get_standard_option('pve-vmid'),
2853     status => {
2854         description => "QEMU process status.",
2855         type => 'string',
2856         enum => ['stopped', 'running'],
2857     },
2858     maxmem => {
2859         description => "Maximum memory in bytes.",
2860         type => 'integer',
2861         optional => 1,
2862         renderer => 'bytes',
2863     },
2864     maxdisk => {
2865         description => "Root disk size in bytes.",
2866         type => 'integer',
2867         optional => 1,
2868         renderer => 'bytes',
2869     },
2870     name => {
2871         description => "VM name.",
2872         type => 'string',
2873         optional => 1,
2874     },
2875     qmpstatus => {
2876         description => "VM run state from the 'query-status' QMP monitor command.",
2877         type => 'string',
2878         optional => 1,
2879     },
2880     pid => {
2881         description => "PID of running qemu process.",
2882         type => 'integer',
2883         optional => 1,
2884     },
2885     uptime => {
2886         description => "Uptime.",
2887         type => 'integer',
2888         optional => 1,
2889         renderer => 'duration',
2890     },
2891     cpus => {
2892         description => "Maximum usable CPUs.",
2893         type => 'number',
2894         optional => 1,
2895     },
2896     lock => {
2897         description => "The current config lock, if any.",
2898         type => 'string',
2899         optional => 1,
2900     },
2901     tags => {
2902         description  => "The current configured tags, if any",
2903         type => 'string',
2904         optional => 1,
2905     },
2906     'running-machine' => {
2907         description => "The currently running machine type (if running).",
2908         type => 'string',
2909         optional => 1,
2910     },
2911     'running-qemu' => {
2912         description => "The currently running QEMU version (if running).",
2913         type => 'string',
2914         optional => 1,
2915     },
2916 };
2917
2918 my $last_proc_pid_stat;
2919
2920 # get VM status information
2921 # This must be fast and should not block ($full == false)
2922 # We only query KVM using QMP if $full == true (this can be slow)
2923 sub vmstatus {
2924     my ($opt_vmid, $full) = @_;
2925
2926     my $res = {};
2927
2928     my $storecfg = PVE::Storage::config();
2929
2930     my $list = vzlist();
2931     my $defaults = load_defaults();
2932
2933     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2934
2935     my $cpucount = $cpuinfo->{cpus} || 1;
2936
2937     foreach my $vmid (keys %$list) {
2938         next if $opt_vmid && ($vmid ne $opt_vmid);
2939
2940         my $conf = PVE::QemuConfig->load_config($vmid);
2941
2942         my $d = { vmid => int($vmid) };
2943         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2944
2945         # fixme: better status?
2946         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2947
2948         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2949         if (defined($size)) {
2950             $d->{disk} = 0; # no info available
2951             $d->{maxdisk} = $size;
2952         } else {
2953             $d->{disk} = 0;
2954             $d->{maxdisk} = 0;
2955         }
2956
2957         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2958             * ($conf->{cores} || $defaults->{cores});
2959         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2960         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2961
2962         $d->{name} = $conf->{name} || "VM $vmid";
2963         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2964             : $defaults->{memory}*(1024*1024);
2965
2966         if ($conf->{balloon}) {
2967             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2968             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2969                 : $defaults->{shares};
2970         }
2971
2972         $d->{uptime} = 0;
2973         $d->{cpu} = 0;
2974         $d->{mem} = 0;
2975
2976         $d->{netout} = 0;
2977         $d->{netin} = 0;
2978
2979         $d->{diskread} = 0;
2980         $d->{diskwrite} = 0;
2981
2982         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2983
2984         $d->{serial} = 1 if conf_has_serial($conf);
2985         $d->{lock} = $conf->{lock} if $conf->{lock};
2986         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2987
2988         $res->{$vmid} = $d;
2989     }
2990
2991     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2992     foreach my $dev (keys %$netdev) {
2993         next if $dev !~ m/^tap([1-9]\d*)i/;
2994         my $vmid = $1;
2995         my $d = $res->{$vmid};
2996         next if !$d;
2997
2998         $d->{netout} += $netdev->{$dev}->{receive};
2999         $d->{netin} += $netdev->{$dev}->{transmit};
3000
3001         if ($full) {
3002             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
3003             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
3004         }
3005
3006     }
3007
3008     my $ctime = gettimeofday;
3009
3010     foreach my $vmid (keys %$list) {
3011
3012         my $d = $res->{$vmid};
3013         my $pid = $d->{pid};
3014         next if !$pid;
3015
3016         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3017         next if !$pstat; # not running
3018
3019         my $used = $pstat->{utime} + $pstat->{stime};
3020
3021         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3022
3023         if ($pstat->{vsize}) {
3024             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3025         }
3026
3027         my $old = $last_proc_pid_stat->{$pid};
3028         if (!$old) {
3029             $last_proc_pid_stat->{$pid} = {
3030                 time => $ctime,
3031                 used => $used,
3032                 cpu => 0,
3033             };
3034             next;
3035         }
3036
3037         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3038
3039         if ($dtime > 1000) {
3040             my $dutime = $used -  $old->{used};
3041
3042             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3043             $last_proc_pid_stat->{$pid} = {
3044                 time => $ctime,
3045                 used => $used,
3046                 cpu => $d->{cpu},
3047             };
3048         } else {
3049             $d->{cpu} = $old->{cpu};
3050         }
3051     }
3052
3053     return $res if !$full;
3054
3055     my $qmpclient = PVE::QMPClient->new();
3056
3057     my $ballooncb = sub {
3058         my ($vmid, $resp) = @_;
3059
3060         my $info = $resp->{'return'};
3061         return if !$info->{max_mem};
3062
3063         my $d = $res->{$vmid};
3064
3065         # use memory assigned to VM
3066         $d->{maxmem} = $info->{max_mem};
3067         $d->{balloon} = $info->{actual};
3068
3069         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3070             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3071             $d->{freemem} = $info->{free_mem};
3072         }
3073
3074         $d->{ballooninfo} = $info;
3075     };
3076
3077     my $blockstatscb = sub {
3078         my ($vmid, $resp) = @_;
3079         my $data = $resp->{'return'} || [];
3080         my $totalrdbytes = 0;
3081         my $totalwrbytes = 0;
3082
3083         for my $blockstat (@$data) {
3084             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3085             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3086
3087             $blockstat->{device} =~ s/drive-//;
3088             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3089         }
3090         $res->{$vmid}->{diskread} = $totalrdbytes;
3091         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3092     };
3093
3094     my $machinecb = sub {
3095         my ($vmid, $resp) = @_;
3096         my $data = $resp->{'return'} || [];
3097
3098         $res->{$vmid}->{'running-machine'} =
3099             PVE::QemuServer::Machine::current_from_query_machines($data);
3100     };
3101
3102     my $versioncb = sub {
3103         my ($vmid, $resp) = @_;
3104         my $data = $resp->{'return'} // {};
3105         my $version = 'unknown';
3106
3107         if (my $v = $data->{qemu}) {
3108             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3109         }
3110
3111         $res->{$vmid}->{'running-qemu'} = $version;
3112     };
3113
3114     my $statuscb = sub {
3115         my ($vmid, $resp) = @_;
3116
3117         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3118         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3119         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3120         # this fails if ballon driver is not loaded, so this must be
3121         # the last commnand (following command are aborted if this fails).
3122         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3123
3124         my $status = 'unknown';
3125         if (!defined($status = $resp->{'return'}->{status})) {
3126             warn "unable to get VM status\n";
3127             return;
3128         }
3129
3130         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3131     };
3132
3133     foreach my $vmid (keys %$list) {
3134         next if $opt_vmid && ($vmid ne $opt_vmid);
3135         next if !$res->{$vmid}->{pid}; # not running
3136         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3137     }
3138
3139     $qmpclient->queue_execute(undef, 2);
3140
3141     foreach my $vmid (keys %$list) {
3142         next if $opt_vmid && ($vmid ne $opt_vmid);
3143         next if !$res->{$vmid}->{pid}; #not running
3144
3145         # we can't use the $qmpclient since it might have already aborted on
3146         # 'query-balloon', but this might also fail for older versions...
3147         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3148         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3149     }
3150
3151     foreach my $vmid (keys %$list) {
3152         next if $opt_vmid && ($vmid ne $opt_vmid);
3153         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3154     }
3155
3156     return $res;
3157 }
3158
3159 sub conf_has_serial {
3160     my ($conf) = @_;
3161
3162     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3163         if ($conf->{"serial$i"}) {
3164             return 1;
3165         }
3166     }
3167
3168     return 0;
3169 }
3170
3171 sub conf_has_audio {
3172     my ($conf, $id) = @_;
3173
3174     $id //= 0;
3175     my $audio = $conf->{"audio$id"};
3176     return if !defined($audio);
3177
3178     my $audioproperties = parse_property_string($audio_fmt, $audio);
3179     my $audiodriver = $audioproperties->{driver} // 'spice';
3180
3181     return {
3182         dev => $audioproperties->{device},
3183         dev_id => "audiodev$id",
3184         backend => $audiodriver,
3185         backend_id => "$audiodriver-backend${id}",
3186     };
3187 }
3188
3189 sub audio_devs {
3190     my ($audio, $audiopciaddr, $machine_version) = @_;
3191
3192     my $devs = [];
3193
3194     my $id = $audio->{dev_id};
3195     my $audiodev = "";
3196     if (min_version($machine_version, 4, 2)) {
3197         $audiodev = ",audiodev=$audio->{backend_id}";
3198     }
3199
3200     if ($audio->{dev} eq 'AC97') {
3201         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3202     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3203         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3204         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3205         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3206     } else {
3207         die "unkown audio device '$audio->{dev}', implement me!";
3208     }
3209
3210     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3211
3212     return $devs;
3213 }
3214
3215 sub get_tpm_paths {
3216     my ($vmid) = @_;
3217     return {
3218         socket => "/var/run/qemu-server/$vmid.swtpm",
3219         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3220     };
3221 }
3222
3223 sub add_tpm_device {
3224     my ($vmid, $devices, $conf) = @_;
3225
3226     return if !$conf->{tpmstate0};
3227
3228     my $paths = get_tpm_paths($vmid);
3229
3230     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3231     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3232     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3233 }
3234
3235 sub start_swtpm {
3236     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3237
3238     return if !$tpmdrive;
3239
3240     my $state;
3241     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3242     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3243     if ($storeid) {
3244         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3245     } else {
3246         $state = $tpm->{file};
3247     }
3248
3249     my $paths = get_tpm_paths($vmid);
3250
3251     # during migration, we will get state from remote
3252     #
3253     if (!$migration) {
3254         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3255         my $setup_cmd = [
3256             "swtpm_setup",
3257             "--tpmstate",
3258             "file://$state",
3259             "--createek",
3260             "--create-ek-cert",
3261             "--create-platform-cert",
3262             "--lock-nvram",
3263             "--config",
3264             "/etc/swtpm_setup.conf", # do not use XDG configs
3265             "--runas",
3266             "0", # force creation as root, error if not possible
3267             "--not-overwrite", # ignore existing state, do not modify
3268         ];
3269
3270         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3271         # TPM 2.0 supports ECC crypto, use if possible
3272         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3273
3274         run_command($setup_cmd, outfunc => sub {
3275             print "swtpm_setup: $1\n";
3276         });
3277     }
3278
3279     # Used to distinguish different invocations in the log.
3280     my $log_prefix = "[id=" . int(time()) . "] ";
3281
3282     my $emulator_cmd = [
3283         "swtpm",
3284         "socket",
3285         "--tpmstate",
3286         "backend-uri=file://$state,mode=0600",
3287         "--ctrl",
3288         "type=unixio,path=$paths->{socket},mode=0600",
3289         "--pid",
3290         "file=$paths->{pid}",
3291         "--terminate", # terminate on QEMU disconnect
3292         "--daemon",
3293         "--log",
3294         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3295     ];
3296     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3297     run_command($emulator_cmd, outfunc => sub { print $1; });
3298
3299     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3300     while (! -e $paths->{pid}) {
3301         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3302         usleep(50_000);
3303     }
3304
3305     # return untainted PID of swtpm daemon so it can be killed on error
3306     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3307     return $1;
3308 }
3309
3310 sub vga_conf_has_spice {
3311     my ($vga) = @_;
3312
3313     my $vgaconf = parse_vga($vga);
3314     my $vgatype = $vgaconf->{type};
3315     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3316
3317     return $1 || 1;
3318 }
3319
3320 sub is_native($) {
3321     my ($arch) = @_;
3322     return get_host_arch() eq $arch;
3323 }
3324
3325 sub get_vm_arch {
3326     my ($conf) = @_;
3327     return $conf->{arch} // get_host_arch();
3328 }
3329
3330 my $default_machines = {
3331     x86_64 => 'pc',
3332     aarch64 => 'virt',
3333 };
3334
3335 sub get_installed_machine_version {
3336     my ($kvmversion) = @_;
3337     $kvmversion = kvm_user_version() if !defined($kvmversion);
3338     $kvmversion =~ m/^(\d+\.\d+)/;
3339     return $1;
3340 }
3341
3342 sub windows_get_pinned_machine_version {
3343     my ($machine, $base_version, $kvmversion) = @_;
3344
3345     my $pin_version = $base_version;
3346     if (!defined($base_version) ||
3347         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3348     ) {
3349         $pin_version = get_installed_machine_version($kvmversion);
3350     }
3351     if (!$machine || $machine eq 'pc') {
3352         $machine = "pc-i440fx-$pin_version";
3353     } elsif ($machine eq 'q35') {
3354         $machine = "pc-q35-$pin_version";
3355     } elsif ($machine eq 'virt') {
3356         $machine = "virt-$pin_version";
3357     } else {
3358         warn "unknown machine type '$machine', not touching that!\n";
3359     }
3360
3361     return $machine;
3362 }
3363
3364 sub get_vm_machine {
3365     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3366
3367     my $machine = $forcemachine || $conf->{machine};
3368
3369     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3370         $kvmversion //= kvm_user_version();
3371         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3372         # layout which confuses windows quite a bit and may result in various regressions..
3373         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3374         if (windows_version($conf->{ostype})) {
3375             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3376         }
3377         $arch //= 'x86_64';
3378         $machine ||= $default_machines->{$arch};
3379         if ($add_pve_version) {
3380             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3381             $machine .= "+pve$pvever";
3382         }
3383     }
3384
3385     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3386         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3387         $machine = $1 if $is_pxe;
3388
3389         # for version-pinned machines that do not include a pve-version (e.g.
3390         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3391         $machine .= '+pve0';
3392
3393         $machine .= '.pxe' if $is_pxe;
3394     }
3395
3396     return $machine;
3397 }
3398
3399 sub get_ovmf_files($$$) {
3400     my ($arch, $efidisk, $smm) = @_;
3401
3402     my $types = $OVMF->{$arch}
3403         or die "no OVMF images known for architecture '$arch'\n";
3404
3405     my $type = 'default';
3406     if ($arch ne "aarch64" && defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3407         $type = $smm ? "4m" : "4m-no-smm";
3408         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3409     }
3410
3411     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3412     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3413     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3414
3415     return ($ovmf_code, $ovmf_vars);
3416 }
3417
3418 my $Arch2Qemu = {
3419     aarch64 => '/usr/bin/qemu-system-aarch64',
3420     x86_64 => '/usr/bin/qemu-system-x86_64',
3421 };
3422 sub get_command_for_arch($) {
3423     my ($arch) = @_;
3424     return '/usr/bin/kvm' if is_native($arch);
3425
3426     my $cmd = $Arch2Qemu->{$arch}
3427         or die "don't know how to emulate architecture '$arch'\n";
3428     return $cmd;
3429 }
3430
3431 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3432 # to use in a QEMU command line (-cpu element), first array_intersect the result
3433 # of query_supported_ with query_understood_. This is necessary because:
3434 #
3435 # a) query_understood_ returns flags the host cannot use and
3436 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3437 #    flags, but CPU settings - with most of them being flags. Those settings
3438 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3439 #
3440 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3441 # expensive. If you need the value returned from this, you can get it much
3442 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3443 # $accel being 'kvm' or 'tcg'.
3444 #
3445 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3446 # changes, automatically populating pmxcfs.
3447 #
3448 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3449 # since kvm and tcg machines support different flags
3450 #
3451 sub query_supported_cpu_flags {
3452     my ($arch) = @_;
3453
3454     $arch //= get_host_arch();
3455     my $default_machine = $default_machines->{$arch};
3456
3457     my $flags = {};
3458
3459     # FIXME: Once this is merged, the code below should work for ARM as well:
3460     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3461     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3462         $arch eq "aarch64";
3463
3464     my $kvm_supported = defined(kvm_version());
3465     my $qemu_cmd = get_command_for_arch($arch);
3466     my $fakevmid = -1;
3467     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3468
3469     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3470     my $query_supported_run_qemu = sub {
3471         my ($kvm) = @_;
3472
3473         my $flags = {};
3474         my $cmd = [
3475             $qemu_cmd,
3476             '-machine', $default_machine,
3477             '-display', 'none',
3478             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3479             '-mon', 'chardev=qmp,mode=control',
3480             '-pidfile', $pidfile,
3481             '-S', '-daemonize'
3482         ];
3483
3484         if (!$kvm) {
3485             push @$cmd, '-accel', 'tcg';
3486         }
3487
3488         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3489         die "QEMU flag querying VM exited with code " . $rc if $rc;
3490
3491         eval {
3492             my $cmd_result = mon_cmd(
3493                 $fakevmid,
3494                 'query-cpu-model-expansion',
3495                 type => 'full',
3496                 model => { name => 'host' }
3497             );
3498
3499             my $props = $cmd_result->{model}->{props};
3500             foreach my $prop (keys %$props) {
3501                 next if $props->{$prop} ne '1';
3502                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3503                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3504                 # We only keep those with underscores, to match /proc/cpuinfo
3505                 $prop =~ s/\.|-/_/g;
3506                 $flags->{$prop} = 1;
3507             }
3508         };
3509         my $err = $@;
3510
3511         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3512         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3513
3514         die $err if $err;
3515
3516         return [ sort keys %$flags ];
3517     };
3518
3519     # We need to query QEMU twice, since KVM and TCG have different supported flags
3520     PVE::QemuConfig->lock_config($fakevmid, sub {
3521         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3522         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3523
3524         if ($kvm_supported) {
3525             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3526             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3527         }
3528     });
3529
3530     return $flags;
3531 }
3532
3533 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3534 my $understood_cpu_flag_dir = "/usr/share/kvm";
3535 sub query_understood_cpu_flags {
3536     my $arch = get_host_arch();
3537     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3538
3539     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3540         if ! -e $filepath;
3541
3542     my $raw = file_get_contents($filepath);
3543     $raw =~ s/^\s+|\s+$//g;
3544     my @flags = split(/\s+/, $raw);
3545
3546     return \@flags;
3547 }
3548
3549 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3550 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3551 my sub should_disable_smm {
3552     my ($conf, $vga, $machine) = @_;
3553
3554     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3555
3556     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3557         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3558 }
3559
3560 my sub print_ovmf_drive_commandlines {
3561     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3562
3563     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3564
3565     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3566
3567     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3568     if ($d) {
3569         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3570         my ($path, $format) = $d->@{'file', 'format'};
3571         if ($storeid) {
3572             $path = PVE::Storage::path($storecfg, $d->{file});
3573             if (!defined($format)) {
3574                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3575                 $format = qemu_img_format($scfg, $volname);
3576             }
3577         } elsif (!defined($format)) {
3578             die "efidisk format must be specified\n";
3579         }
3580         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3581         if ($path =~ m/^rbd:/) {
3582             $var_drive_str .= ',cache=writeback';
3583             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3584         }
3585         $var_drive_str .= ",format=$format,file=$path";
3586
3587         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3588         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3589     } else {
3590         log_warn("no efidisk configured! Using temporary efivars disk.");
3591         my $path = "/tmp/$vmid-ovmf.fd";
3592         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3593         $var_drive_str .= ",format=raw,file=$path";
3594         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3595     }
3596
3597     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3598 }
3599
3600 sub config_to_command {
3601     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3602         $pbs_backing) = @_;
3603
3604     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3605     my $devices = [];
3606     my $bridges = {};
3607     my $ostype = $conf->{ostype};
3608     my $winversion = windows_version($ostype);
3609     my $kvm = $conf->{kvm};
3610     my $nodename = nodename();
3611
3612     my $arch = get_vm_arch($conf);
3613     my $kvm_binary = get_command_for_arch($arch);
3614     my $kvmver = kvm_user_version($kvm_binary);
3615
3616     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3617         $kvmver //= "undefined";
3618         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3619     }
3620
3621     my $add_pve_version = min_version($kvmver, 4, 1);
3622
3623     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3624     my $machine_version = extract_version($machine_type, $kvmver);
3625     $kvm //= 1 if is_native($arch);
3626
3627     $machine_version =~ m/(\d+)\.(\d+)/;
3628     my ($machine_major, $machine_minor) = ($1, $2);
3629
3630     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3631         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3632     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3633         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3634             ." please upgrade node '$nodename'\n"
3635     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3636         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3637         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3638             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3639             ." node '$nodename'\n";
3640     }
3641
3642     # if a specific +pve version is required for a feature, use $version_guard
3643     # instead of min_version to allow machines to be run with the minimum
3644     # required version
3645     my $required_pve_version = 0;
3646     my $version_guard = sub {
3647         my ($major, $minor, $pve) = @_;
3648         return 0 if !min_version($machine_version, $major, $minor, $pve);
3649         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3650         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3651         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3652         return 1;
3653     };
3654
3655     if ($kvm && !defined kvm_version()) {
3656         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3657             ." or enable in BIOS.\n";
3658     }
3659
3660     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3661     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3662     my $use_old_bios_files = undef;
3663     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3664
3665     my $cmd = [];
3666     if ($conf->{affinity}) {
3667         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3668     }
3669
3670     push @$cmd, $kvm_binary;
3671
3672     push @$cmd, '-id', $vmid;
3673
3674     my $vmname = $conf->{name} || "vm$vmid";
3675
3676     push @$cmd, '-name', "$vmname,debug-threads=on";
3677
3678     push @$cmd, '-no-shutdown';
3679
3680     my $use_virtio = 0;
3681
3682     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3683     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3684     push @$cmd, '-mon', "chardev=qmp,mode=control";
3685
3686     if (min_version($machine_version, 2, 12)) {
3687         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3688         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3689     }
3690
3691     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3692
3693     push @$cmd, '-daemonize';
3694
3695     if ($conf->{smbios1}) {
3696         my $smbios_conf = parse_smbios1($conf->{smbios1});
3697         if ($smbios_conf->{base64}) {
3698             # Do not pass base64 flag to qemu
3699             delete $smbios_conf->{base64};
3700             my $smbios_string = "";
3701             foreach my $key (keys %$smbios_conf) {
3702                 my $value;
3703                 if ($key eq "uuid") {
3704                     $value = $smbios_conf->{uuid}
3705                 } else {
3706                     $value = decode_base64($smbios_conf->{$key});
3707                 }
3708                 # qemu accepts any binary data, only commas need escaping by double comma
3709                 $value =~ s/,/,,/g;
3710                 $smbios_string .= "," . $key . "=" . $value if $value;
3711             }
3712             push @$cmd, '-smbios', "type=1" . $smbios_string;
3713         } else {
3714             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3715         }
3716     }
3717
3718     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3719         my ($code_drive_str, $var_drive_str) =
3720             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3721         push $cmd->@*, '-drive', $code_drive_str;
3722         push $cmd->@*, '-drive', $var_drive_str;
3723     }
3724
3725     if ($q35) { # tell QEMU to load q35 config early
3726         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3727         if (min_version($machine_version, 4, 0)) {
3728             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3729         } else {
3730             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3731         }
3732     }
3733
3734     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3735         push @$cmd, $fixups->@*;
3736     }
3737
3738     if ($conf->{vmgenid}) {
3739         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3740     }
3741
3742     # add usb controllers
3743     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3744         $conf, $bridges, $arch, $machine_type, $machine_version);
3745     push @$devices, @usbcontrollers if @usbcontrollers;
3746     my $vga = parse_vga($conf->{vga});
3747
3748     my $qxlnum = vga_conf_has_spice($conf->{vga});
3749     $vga->{type} = 'qxl' if $qxlnum;
3750
3751     if (!$vga->{type}) {
3752         if ($arch eq 'aarch64') {
3753             $vga->{type} = 'virtio';
3754         } elsif (min_version($machine_version, 2, 9)) {
3755             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3756         } else {
3757             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3758         }
3759     }
3760
3761     # enable absolute mouse coordinates (needed by vnc)
3762     my $tablet = $conf->{tablet};
3763     if (!defined($tablet)) {
3764         $tablet = $defaults->{tablet};
3765         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3766         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3767     }
3768
3769     if ($tablet) {
3770         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3771         my $kbd = print_keyboarddevice_full($conf, $arch);
3772         push @$devices, '-device', $kbd if defined($kbd);
3773     }
3774
3775     my $bootorder = device_bootorder($conf);
3776
3777     # host pci device passthrough
3778     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3779         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3780
3781     # usb devices
3782     my $usb_dev_features = {};
3783     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3784
3785     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3786         $conf, $usb_dev_features, $bootorder, $machine_version);
3787     push @$devices, @usbdevices if @usbdevices;
3788
3789     # serial devices
3790     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3791         my $path = $conf->{"serial$i"} or next;
3792         if ($path eq 'socket') {
3793             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3794             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3795             # On aarch64, serial0 is the UART device. QEMU only allows
3796             # connecting UART devices via the '-serial' command line, as
3797             # the device has a fixed slot on the hardware...
3798             if ($arch eq 'aarch64' && $i == 0) {
3799                 push @$devices, '-serial', "chardev:serial$i";
3800             } else {
3801                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3802             }
3803         } else {
3804             die "no such serial device\n" if ! -c $path;
3805             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3806             push @$devices, '-device', "isa-serial,chardev=serial$i";
3807         }
3808     }
3809
3810     # parallel devices
3811     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3812         if (my $path = $conf->{"parallel$i"}) {
3813             die "no such parallel device\n" if ! -c $path;
3814             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3815             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3816             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3817         }
3818     }
3819
3820     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3821         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3822         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3823         push @$devices, @$audio_devs;
3824     }
3825
3826     add_tpm_device($vmid, $devices, $conf);
3827
3828     my $sockets = 1;
3829     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3830     $sockets = $conf->{sockets} if  $conf->{sockets};
3831
3832     my $cores = $conf->{cores} || 1;
3833
3834     my $maxcpus = $sockets * $cores;
3835
3836     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3837
3838     my $allowed_vcpus = $cpuinfo->{cpus};
3839
3840     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3841
3842     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3843         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3844         for (my $i = 2; $i <= $vcpus; $i++)  {
3845             my $cpustr = print_cpu_device($conf,$i);
3846             push @$cmd, '-device', $cpustr;
3847         }
3848
3849     } else {
3850
3851         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3852     }
3853     push @$cmd, '-nodefaults';
3854
3855     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3856
3857     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3858
3859     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3860
3861     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3862         push @$devices, '-device', print_vga_device(
3863             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3864
3865         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3866
3867         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3868         push @$cmd,  '-vnc', "unix:$socket,password=on";
3869     } else {
3870         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3871         push @$cmd, '-nographic';
3872     }
3873
3874     # time drift fix
3875     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3876     my $useLocaltime = $conf->{localtime};
3877
3878     if ($winversion >= 5) { # windows
3879         $useLocaltime = 1 if !defined($conf->{localtime});
3880
3881         # use time drift fix when acpi is enabled
3882         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3883             $tdf = 1 if !defined($conf->{tdf});
3884         }
3885     }
3886
3887     if ($winversion >= 6) {
3888         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3889         push @$machineFlags, 'hpet=off';
3890     }
3891
3892     push @$rtcFlags, 'driftfix=slew' if $tdf;
3893
3894     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3895         push @$rtcFlags, "base=$conf->{startdate}";
3896     } elsif ($useLocaltime) {
3897         push @$rtcFlags, 'base=localtime';
3898     }
3899
3900     if ($forcecpu) {
3901         push @$cmd, '-cpu', $forcecpu;
3902     } else {
3903         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3904     }
3905
3906     PVE::QemuServer::Memory::config(
3907         $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory}, $cmd);
3908
3909     push @$cmd, '-S' if $conf->{freeze};
3910
3911     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3912
3913     my $guest_agent = parse_guest_agent($conf);
3914
3915     if ($guest_agent->{enabled}) {
3916         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3917         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3918
3919         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3920             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3921             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3922             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3923         } elsif ($guest_agent->{type} eq 'isa') {
3924             push @$devices, '-device', "isa-serial,chardev=qga0";
3925         }
3926     }
3927
3928     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3929     if ($rng && $version_guard->(4, 1, 2)) {
3930         check_rng_source($rng->{source});
3931
3932         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3933         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3934         my $limiter_str = "";
3935         if ($max_bytes) {
3936             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3937         }
3938
3939         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3940         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3941         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3942     }
3943
3944     my $spice_port;
3945
3946     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3947         if ($qxlnum > 1) {
3948             if ($winversion){
3949                 for (my $i = 1; $i < $qxlnum; $i++){
3950                     push @$devices, '-device', print_vga_device(
3951                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3952                 }
3953             } else {
3954                 # assume other OS works like Linux
3955                 my ($ram, $vram) = ("134217728", "67108864");
3956                 if ($vga->{memory}) {
3957                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3958                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3959                 }
3960                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3961                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3962             }
3963         }
3964
3965         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3966
3967         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3968         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3969         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3970
3971         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3972         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3973         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3974
3975         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3976         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3977
3978         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3979         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3980         if ($spice_enhancement->{foldersharing}) {
3981             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3982             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3983         }
3984
3985         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3986         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3987             if $spice_enhancement->{videostreaming};
3988
3989         push @$devices, '-spice', "$spice_opts";
3990     }
3991
3992     # enable balloon by default, unless explicitly disabled
3993     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3994         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3995         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3996         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3997         push @$devices, '-device', $ballooncmd;
3998     }
3999
4000     if ($conf->{watchdog}) {
4001         my $wdopts = parse_watchdog($conf->{watchdog});
4002         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4003         my $watchdog = $wdopts->{model} || 'i6300esb';
4004         push @$devices, '-device', "$watchdog$pciaddr";
4005         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4006     }
4007
4008     my $vollist = [];
4009     my $scsicontroller = {};
4010     my $ahcicontroller = {};
4011     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4012
4013     # Add iscsi initiator name if available
4014     if (my $initiator = get_initiator_name()) {
4015         push @$devices, '-iscsi', "initiator-name=$initiator";
4016     }
4017
4018     PVE::QemuConfig->foreach_volume($conf, sub {
4019         my ($ds, $drive) = @_;
4020
4021         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4022             check_volume_storage_type($storecfg, $drive->{file});
4023             push @$vollist, $drive->{file};
4024         }
4025
4026         # ignore efidisk here, already added in bios/fw handling code above
4027         return if $drive->{interface} eq 'efidisk';
4028         # similar for TPM
4029         return if $drive->{interface} eq 'tpmstate';
4030
4031         $use_virtio = 1 if $ds =~ m/^virtio/;
4032
4033         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4034
4035         if ($drive->{interface} eq 'virtio'){
4036            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4037         }
4038
4039         if ($drive->{interface} eq 'scsi') {
4040
4041             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4042
4043             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4044                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4045
4046             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4047             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4048
4049             my $iothread = '';
4050             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4051                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4052                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4053             } elsif ($drive->{iothread}) {
4054                 log_warn(
4055                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4056                 );
4057             }
4058
4059             my $queues = '';
4060             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4061                 $queues = ",num_queues=$drive->{queues}";
4062             }
4063
4064             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4065                 if !$scsicontroller->{$controller};
4066             $scsicontroller->{$controller}=1;
4067         }
4068
4069         if ($drive->{interface} eq 'sata') {
4070             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4071             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4072             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4073                 if !$ahcicontroller->{$controller};
4074             $ahcicontroller->{$controller}=1;
4075         }
4076
4077         my $pbs_conf = $pbs_backing->{$ds};
4078         my $pbs_name = undef;
4079         if ($pbs_conf) {
4080             $pbs_name = "drive-$ds-pbs";
4081             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4082         }
4083
4084         my $drive_cmd = print_drive_commandline_full(
4085             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4086
4087         # extra protection for templates, but SATA and IDE don't support it..
4088         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4089
4090         push @$devices, '-drive',$drive_cmd;
4091         push @$devices, '-device', print_drivedevice_full(
4092             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4093     });
4094
4095     for (my $i = 0; $i < $MAX_NETS; $i++) {
4096         my $netname = "net$i";
4097
4098         next if !$conf->{$netname};
4099         my $d = parse_net($conf->{$netname});
4100         next if !$d;
4101         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4102
4103         $use_virtio = 1 if $d->{model} eq 'virtio';
4104
4105         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4106
4107         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4108         push @$devices, '-netdev', $netdevfull;
4109
4110         my $netdevicefull = print_netdevice_full(
4111             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4112
4113         push @$devices, '-device', $netdevicefull;
4114     }
4115
4116     if ($conf->{ivshmem}) {
4117         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4118
4119         my $bus;
4120         if ($q35) {
4121             $bus = print_pcie_addr("ivshmem");
4122         } else {
4123             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4124         }
4125
4126         my $ivshmem_name = $ivshmem->{name} // $vmid;
4127         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4128
4129         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4130         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4131             .",size=$ivshmem->{size}M";
4132     }
4133
4134     # pci.4 is nested in pci.1
4135     $bridges->{1} = 1 if $bridges->{4};
4136
4137     if (!$q35) { # add pci bridges
4138         if (min_version($machine_version, 2, 3)) {
4139            $bridges->{1} = 1;
4140            $bridges->{2} = 1;
4141         }
4142         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4143     }
4144
4145     for my $k (sort {$b cmp $a} keys %$bridges) {
4146         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4147
4148         my $k_name = $k;
4149         if ($k == 2 && $legacy_igd) {
4150             $k_name = "$k-igd";
4151         }
4152         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4153         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4154
4155         if ($q35) { # add after -readconfig pve-q35.cfg
4156             splice @$devices, 2, 0, '-device', $devstr;
4157         } else {
4158             unshift @$devices, '-device', $devstr if $k > 0;
4159         }
4160     }
4161
4162     if (!$kvm) {
4163         push @$machineFlags, 'accel=tcg';
4164     }
4165
4166     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4167
4168     my $machine_type_min = $machine_type;
4169     if ($add_pve_version) {
4170         $machine_type_min =~ s/\+pve\d+$//;
4171         $machine_type_min .= "+pve$required_pve_version";
4172     }
4173     push @$machineFlags, "type=${machine_type_min}";
4174
4175     push @$cmd, @$devices;
4176     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4177     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4178     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4179
4180     if (my $vmstate = $conf->{vmstate}) {
4181         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4182         push @$vollist, $vmstate;
4183         push @$cmd, '-loadstate', $statepath;
4184         print "activating and using '$vmstate' as vmstate\n";
4185     }
4186
4187     if (PVE::QemuConfig->is_template($conf)) {
4188         # needed to workaround base volumes being read-only
4189         push @$cmd, '-snapshot';
4190     }
4191
4192     # add custom args
4193     if ($conf->{args}) {
4194         my $aa = PVE::Tools::split_args($conf->{args});
4195         push @$cmd, @$aa;
4196     }
4197
4198     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4199 }
4200
4201 sub check_rng_source {
4202     my ($source) = @_;
4203
4204     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4205     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4206         if ! -e $source;
4207
4208     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4209     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4210         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4211         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4212         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4213             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4214             ." to the host.\n";
4215     }
4216 }
4217
4218 sub spice_port {
4219     my ($vmid) = @_;
4220
4221     my $res = mon_cmd($vmid, 'query-spice');
4222
4223     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4224 }
4225
4226 sub vm_devices_list {
4227     my ($vmid) = @_;
4228
4229     my $res = mon_cmd($vmid, 'query-pci');
4230     my $devices_to_check = [];
4231     my $devices = {};
4232     foreach my $pcibus (@$res) {
4233         push @$devices_to_check, @{$pcibus->{devices}},
4234     }
4235
4236     while (@$devices_to_check) {
4237         my $to_check = [];
4238         for my $d (@$devices_to_check) {
4239             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4240             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4241
4242             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4243             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4244         }
4245         $devices_to_check = $to_check;
4246     }
4247
4248     my $resblock = mon_cmd($vmid, 'query-block');
4249     foreach my $block (@$resblock) {
4250         if($block->{device} =~ m/^drive-(\S+)/){
4251                 $devices->{$1} = 1;
4252         }
4253     }
4254
4255     my $resmice = mon_cmd($vmid, 'query-mice');
4256     foreach my $mice (@$resmice) {
4257         if ($mice->{name} eq 'QEMU HID Tablet') {
4258             $devices->{tablet} = 1;
4259             last;
4260         }
4261     }
4262
4263     # for usb devices there is no query-usb
4264     # but we can iterate over the entries in
4265     # qom-list path=/machine/peripheral
4266     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4267     foreach my $per (@$resperipheral) {
4268         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4269             $devices->{$per->{name}} = 1;
4270         }
4271     }
4272
4273     return $devices;
4274 }
4275
4276 sub vm_deviceplug {
4277     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4278
4279     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4280
4281     my $devices_list = vm_devices_list($vmid);
4282     return 1 if defined($devices_list->{$deviceid});
4283
4284     # add PCI bridge if we need it for the device
4285     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4286
4287     if ($deviceid eq 'tablet') {
4288         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4289     } elsif ($deviceid eq 'keyboard') {
4290         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4291     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4292         my $id = $1;
4293         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4294         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4295     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4296         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4297     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4298         qemu_iothread_add($vmid, $deviceid, $device);
4299
4300         qemu_driveadd($storecfg, $vmid, $device);
4301         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4302
4303         qemu_deviceadd($vmid, $devicefull);
4304         eval { qemu_deviceaddverify($vmid, $deviceid); };
4305         if (my $err = $@) {
4306             eval { qemu_drivedel($vmid, $deviceid); };
4307             warn $@ if $@;
4308             die $err;
4309         }
4310     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4311         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4312         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4313         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4314
4315         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4316
4317         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4318             qemu_iothread_add($vmid, $deviceid, $device);
4319             $devicefull .= ",iothread=iothread-$deviceid";
4320         }
4321
4322         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4323             $devicefull .= ",num_queues=$device->{queues}";
4324         }
4325
4326         qemu_deviceadd($vmid, $devicefull);
4327         qemu_deviceaddverify($vmid, $deviceid);
4328     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4329         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4330         qemu_driveadd($storecfg, $vmid, $device);
4331
4332         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4333         eval { qemu_deviceadd($vmid, $devicefull); };
4334         if (my $err = $@) {
4335             eval { qemu_drivedel($vmid, $deviceid); };
4336             warn $@ if $@;
4337             die $err;
4338         }
4339     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4340         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4341
4342         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4343         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4344         my $use_old_bios_files = undef;
4345         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4346
4347         my $netdevicefull = print_netdevice_full(
4348             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4349         qemu_deviceadd($vmid, $netdevicefull);
4350         eval {
4351             qemu_deviceaddverify($vmid, $deviceid);
4352             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4353         };
4354         if (my $err = $@) {
4355             eval { qemu_netdevdel($vmid, $deviceid); };
4356             warn $@ if $@;
4357             die $err;
4358         }
4359     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4360         my $bridgeid = $2;
4361         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4362         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4363
4364         qemu_deviceadd($vmid, $devicefull);
4365         qemu_deviceaddverify($vmid, $deviceid);
4366     } else {
4367         die "can't hotplug device '$deviceid'\n";
4368     }
4369
4370     return 1;
4371 }
4372
4373 # fixme: this should raise exceptions on error!
4374 sub vm_deviceunplug {
4375     my ($vmid, $conf, $deviceid) = @_;
4376
4377     my $devices_list = vm_devices_list($vmid);
4378     return 1 if !defined($devices_list->{$deviceid});
4379
4380     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4381     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4382
4383     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4384         qemu_devicedel($vmid, $deviceid);
4385     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4386         qemu_devicedel($vmid, $deviceid);
4387         qemu_devicedelverify($vmid, $deviceid);
4388     } elsif ($deviceid =~ m/^usb\d+$/) {
4389         qemu_devicedel($vmid, $deviceid);
4390         qemu_devicedelverify($vmid, $deviceid);
4391     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4392         my $device = parse_drive($deviceid, $conf->{$deviceid});
4393
4394         qemu_devicedel($vmid, $deviceid);
4395         qemu_devicedelverify($vmid, $deviceid);
4396         qemu_drivedel($vmid, $deviceid);
4397         qemu_iothread_del($vmid, $deviceid, $device);
4398     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4399         qemu_devicedel($vmid, $deviceid);
4400         qemu_devicedelverify($vmid, $deviceid);
4401     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4402         my $device = parse_drive($deviceid, $conf->{$deviceid});
4403
4404         qemu_devicedel($vmid, $deviceid);
4405         qemu_devicedelverify($vmid, $deviceid);
4406         qemu_drivedel($vmid, $deviceid);
4407         qemu_deletescsihw($conf, $vmid, $deviceid);
4408
4409         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4410             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4411     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4412         qemu_devicedel($vmid, $deviceid);
4413         qemu_devicedelverify($vmid, $deviceid);
4414         qemu_netdevdel($vmid, $deviceid);
4415     } else {
4416         die "can't unplug device '$deviceid'\n";
4417     }
4418
4419     return 1;
4420 }
4421
4422 sub qemu_spice_usbredir_chardev_add {
4423     my ($vmid, $id) = @_;
4424
4425     mon_cmd($vmid, "chardev-add" , (
4426         id => $id,
4427         backend => {
4428             type => 'spicevmc',
4429             data => {
4430                 type => "usbredir",
4431             },
4432         },
4433     ));
4434 }
4435
4436 sub qemu_deviceadd {
4437     my ($vmid, $devicefull) = @_;
4438
4439     $devicefull = "driver=".$devicefull;
4440     my %options =  split(/[=,]/, $devicefull);
4441
4442     mon_cmd($vmid, "device_add" , %options);
4443 }
4444
4445 sub qemu_devicedel {
4446     my ($vmid, $deviceid) = @_;
4447
4448     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4449 }
4450
4451 sub qemu_iothread_add {
4452     my ($vmid, $deviceid, $device) = @_;
4453
4454     if ($device->{iothread}) {
4455         my $iothreads = vm_iothreads_list($vmid);
4456         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4457     }
4458 }
4459
4460 sub qemu_iothread_del {
4461     my ($vmid, $deviceid, $device) = @_;
4462
4463     if ($device->{iothread}) {
4464         my $iothreads = vm_iothreads_list($vmid);
4465         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4466     }
4467 }
4468
4469 sub qemu_objectadd {
4470     my ($vmid, $objectid, $qomtype) = @_;
4471
4472     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4473
4474     return 1;
4475 }
4476
4477 sub qemu_objectdel {
4478     my ($vmid, $objectid) = @_;
4479
4480     mon_cmd($vmid, "object-del", id => $objectid);
4481
4482     return 1;
4483 }
4484
4485 sub qemu_driveadd {
4486     my ($storecfg, $vmid, $device) = @_;
4487
4488     my $kvmver = get_running_qemu_version($vmid);
4489     my $io_uring = min_version($kvmver, 6, 0);
4490     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4491     $drive =~ s/\\/\\\\/g;
4492     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4493
4494     # If the command succeeds qemu prints: "OK"
4495     return 1 if $ret =~ m/OK/s;
4496
4497     die "adding drive failed: $ret\n";
4498 }
4499
4500 sub qemu_drivedel {
4501     my ($vmid, $deviceid) = @_;
4502
4503     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4504     $ret =~ s/^\s+//;
4505
4506     return 1 if $ret eq "";
4507
4508     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4509     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4510
4511     die "deleting drive $deviceid failed : $ret\n";
4512 }
4513
4514 sub qemu_deviceaddverify {
4515     my ($vmid, $deviceid) = @_;
4516
4517     for (my $i = 0; $i <= 5; $i++) {
4518          my $devices_list = vm_devices_list($vmid);
4519          return 1 if defined($devices_list->{$deviceid});
4520          sleep 1;
4521     }
4522
4523     die "error on hotplug device '$deviceid'\n";
4524 }
4525
4526
4527 sub qemu_devicedelverify {
4528     my ($vmid, $deviceid) = @_;
4529
4530     # need to verify that the device is correctly removed as device_del
4531     # is async and empty return is not reliable
4532
4533     for (my $i = 0; $i <= 5; $i++) {
4534          my $devices_list = vm_devices_list($vmid);
4535          return 1 if !defined($devices_list->{$deviceid});
4536          sleep 1;
4537     }
4538
4539     die "error on hot-unplugging device '$deviceid'\n";
4540 }
4541
4542 sub qemu_findorcreatescsihw {
4543     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4544
4545     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4546
4547     my $scsihwid="$controller_prefix$controller";
4548     my $devices_list = vm_devices_list($vmid);
4549
4550     if (!defined($devices_list->{$scsihwid})) {
4551         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4552     }
4553
4554     return 1;
4555 }
4556
4557 sub qemu_deletescsihw {
4558     my ($conf, $vmid, $opt) = @_;
4559
4560     my $device = parse_drive($opt, $conf->{$opt});
4561
4562     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4563         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4564         return 1;
4565     }
4566
4567     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4568
4569     my $devices_list = vm_devices_list($vmid);
4570     foreach my $opt (keys %{$devices_list}) {
4571         if (is_valid_drivename($opt)) {
4572             my $drive = parse_drive($opt, $conf->{$opt});
4573             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4574                 return 1;
4575             }
4576         }
4577     }
4578
4579     my $scsihwid="scsihw$controller";
4580
4581     vm_deviceunplug($vmid, $conf, $scsihwid);
4582
4583     return 1;
4584 }
4585
4586 sub qemu_add_pci_bridge {
4587     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4588
4589     my $bridges = {};
4590
4591     my $bridgeid;
4592
4593     print_pci_addr($device, $bridges, $arch, $machine_type);
4594
4595     while (my ($k, $v) = each %$bridges) {
4596         $bridgeid = $k;
4597     }
4598     return 1 if !defined($bridgeid) || $bridgeid < 1;
4599
4600     my $bridge = "pci.$bridgeid";
4601     my $devices_list = vm_devices_list($vmid);
4602
4603     if (!defined($devices_list->{$bridge})) {
4604         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4605     }
4606
4607     return 1;
4608 }
4609
4610 sub qemu_set_link_status {
4611     my ($vmid, $device, $up) = @_;
4612
4613     mon_cmd($vmid, "set_link", name => $device,
4614                up => $up ? JSON::true : JSON::false);
4615 }
4616
4617 sub qemu_netdevadd {
4618     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4619
4620     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4621     my %options =  split(/[=,]/, $netdev);
4622
4623     if (defined(my $vhost = $options{vhost})) {
4624         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4625     }
4626
4627     if (defined(my $queues = $options{queues})) {
4628         $options{queues} = $queues + 0;
4629     }
4630
4631     mon_cmd($vmid, "netdev_add",  %options);
4632     return 1;
4633 }
4634
4635 sub qemu_netdevdel {
4636     my ($vmid, $deviceid) = @_;
4637
4638     mon_cmd($vmid, "netdev_del", id => $deviceid);
4639 }
4640
4641 sub qemu_usb_hotplug {
4642     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4643
4644     return if !$device;
4645
4646     # remove the old one first
4647     vm_deviceunplug($vmid, $conf, $deviceid);
4648
4649     # check if xhci controller is necessary and available
4650     my $devicelist = vm_devices_list($vmid);
4651
4652     if (!$devicelist->{xhci}) {
4653         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4654         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4655     }
4656
4657     # add the new one
4658     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4659 }
4660
4661 sub qemu_cpu_hotplug {
4662     my ($vmid, $conf, $vcpus) = @_;
4663
4664     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4665
4666     my $sockets = 1;
4667     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4668     $sockets = $conf->{sockets} if  $conf->{sockets};
4669     my $cores = $conf->{cores} || 1;
4670     my $maxcpus = $sockets * $cores;
4671
4672     $vcpus = $maxcpus if !$vcpus;
4673
4674     die "you can't add more vcpus than maxcpus\n"
4675         if $vcpus > $maxcpus;
4676
4677     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4678
4679     if ($vcpus < $currentvcpus) {
4680
4681         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4682
4683             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4684                 qemu_devicedel($vmid, "cpu$i");
4685                 my $retry = 0;
4686                 my $currentrunningvcpus = undef;
4687                 while (1) {
4688                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4689                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4690                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4691                     $retry++;
4692                     sleep 1;
4693                 }
4694                 #update conf after each succesfull cpu unplug
4695                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4696                 PVE::QemuConfig->write_config($vmid, $conf);
4697             }
4698         } else {
4699             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4700         }
4701
4702         return;
4703     }
4704
4705     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4706     die "vcpus in running vm does not match its configuration\n"
4707         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4708
4709     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4710
4711         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4712             my $cpustr = print_cpu_device($conf, $i);
4713             qemu_deviceadd($vmid, $cpustr);
4714
4715             my $retry = 0;
4716             my $currentrunningvcpus = undef;
4717             while (1) {
4718                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4719                 last if scalar(@{$currentrunningvcpus}) == $i;
4720                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4721                 sleep 1;
4722                 $retry++;
4723             }
4724             #update conf after each succesfull cpu hotplug
4725             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4726             PVE::QemuConfig->write_config($vmid, $conf);
4727         }
4728     } else {
4729
4730         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4731             mon_cmd($vmid, "cpu-add", id => int($i));
4732         }
4733     }
4734 }
4735
4736 sub qemu_block_set_io_throttle {
4737     my ($vmid, $deviceid,
4738         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4739         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4740         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4741         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4742
4743     return if !check_running($vmid) ;
4744
4745     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4746         bps => int($bps),
4747         bps_rd => int($bps_rd),
4748         bps_wr => int($bps_wr),
4749         iops => int($iops),
4750         iops_rd => int($iops_rd),
4751         iops_wr => int($iops_wr),
4752         bps_max => int($bps_max),
4753         bps_rd_max => int($bps_rd_max),
4754         bps_wr_max => int($bps_wr_max),
4755         iops_max => int($iops_max),
4756         iops_rd_max => int($iops_rd_max),
4757         iops_wr_max => int($iops_wr_max),
4758         bps_max_length => int($bps_max_length),
4759         bps_rd_max_length => int($bps_rd_max_length),
4760         bps_wr_max_length => int($bps_wr_max_length),
4761         iops_max_length => int($iops_max_length),
4762         iops_rd_max_length => int($iops_rd_max_length),
4763         iops_wr_max_length => int($iops_wr_max_length),
4764     );
4765
4766 }
4767
4768 sub qemu_block_resize {
4769     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4770
4771     my $running = check_running($vmid);
4772
4773     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4774
4775     return if !$running;
4776
4777     my $padding = (1024 - $size % 1024) % 1024;
4778     $size = $size + $padding;
4779
4780     mon_cmd(
4781         $vmid,
4782         "block_resize",
4783         device => $deviceid,
4784         size => int($size),
4785         timeout => 60,
4786     );
4787 }
4788
4789 sub qemu_volume_snapshot {
4790     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4791
4792     my $running = check_running($vmid);
4793
4794     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4795         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4796     } else {
4797         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4798     }
4799 }
4800
4801 sub qemu_volume_snapshot_delete {
4802     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4803
4804     my $running = check_running($vmid);
4805
4806     if($running) {
4807
4808         $running = undef;
4809         my $conf = PVE::QemuConfig->load_config($vmid);
4810         PVE::QemuConfig->foreach_volume($conf, sub {
4811             my ($ds, $drive) = @_;
4812             $running = 1 if $drive->{file} eq $volid;
4813         });
4814     }
4815
4816     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4817         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4818     } else {
4819         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4820     }
4821 }
4822
4823 sub set_migration_caps {
4824     my ($vmid, $savevm) = @_;
4825
4826     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4827
4828     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4829     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4830
4831     my $cap_ref = [];
4832
4833     my $enabled_cap = {
4834         "auto-converge" => 1,
4835         "xbzrle" => 1,
4836         "x-rdma-pin-all" => 0,
4837         "zero-blocks" => 0,
4838         "compress" => 0,
4839         "dirty-bitmaps" => $dirty_bitmaps,
4840     };
4841
4842     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4843
4844     for my $supported_capability (@$supported_capabilities) {
4845         push @$cap_ref, {
4846             capability => $supported_capability->{capability},
4847             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4848         };
4849     }
4850
4851     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4852 }
4853
4854 sub foreach_volid {
4855     my ($conf, $func, @param) = @_;
4856
4857     my $volhash = {};
4858
4859     my $test_volid = sub {
4860         my ($key, $drive, $snapname, $pending) = @_;
4861
4862         my $volid = $drive->{file};
4863         return if !$volid;
4864
4865         $volhash->{$volid}->{cdrom} //= 1;
4866         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4867
4868         my $replicate = $drive->{replicate} // 1;
4869         $volhash->{$volid}->{replicate} //= 0;
4870         $volhash->{$volid}->{replicate} = 1 if $replicate;
4871
4872         $volhash->{$volid}->{shared} //= 0;
4873         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4874
4875         $volhash->{$volid}->{is_unused} //= 0;
4876         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4877
4878         $volhash->{$volid}->{is_attached} //= 0;
4879         $volhash->{$volid}->{is_attached} = 1
4880             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4881
4882         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4883             if defined($snapname);
4884
4885         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4886
4887         my $size = $drive->{size};
4888         $volhash->{$volid}->{size} //= $size if $size;
4889
4890         $volhash->{$volid}->{is_vmstate} //= 0;
4891         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4892
4893         $volhash->{$volid}->{is_tpmstate} //= 0;
4894         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4895
4896         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4897     };
4898
4899     my $include_opts = {
4900         extra_keys => ['vmstate'],
4901         include_unused => 1,
4902     };
4903
4904     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4905
4906     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4907         if defined($conf->{pending}) && $conf->{pending}->%*;
4908
4909     foreach my $snapname (keys %{$conf->{snapshots}}) {
4910         my $snap = $conf->{snapshots}->{$snapname};
4911         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4912     }
4913
4914     foreach my $volid (keys %$volhash) {
4915         &$func($volid, $volhash->{$volid}, @param);
4916     }
4917 }
4918
4919 my $fast_plug_option = {
4920     'description' => 1,
4921     'hookscript' => 1,
4922     'lock' => 1,
4923     'migrate_downtime' => 1,
4924     'migrate_speed' => 1,
4925     'name' => 1,
4926     'onboot' => 1,
4927     'protection' => 1,
4928     'shares' => 1,
4929     'startup' => 1,
4930     'tags' => 1,
4931     'vmstatestorage' => 1,
4932 };
4933
4934 for my $opt (keys %$confdesc_cloudinit) {
4935     $fast_plug_option->{$opt} = 1;
4936 };
4937
4938 # hotplug changes in [PENDING]
4939 # $selection hash can be used to only apply specified options, for
4940 # example: { cores => 1 } (only apply changed 'cores')
4941 # $errors ref is used to return error messages
4942 sub vmconfig_hotplug_pending {
4943     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4944
4945     my $defaults = load_defaults();
4946     my $arch = get_vm_arch($conf);
4947     my $machine_type = get_vm_machine($conf, undef, $arch);
4948
4949     # commit values which do not have any impact on running VM first
4950     # Note: those option cannot raise errors, we we do not care about
4951     # $selection and always apply them.
4952
4953     my $add_error = sub {
4954         my ($opt, $msg) = @_;
4955         $errors->{$opt} = "hotplug problem - $msg";
4956     };
4957
4958     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4959
4960     my $cloudinit_record_changed = sub {
4961         my ($conf, $opt, $old, $new) = @_;
4962         return if !$cloudinit_pending_properties->{$opt};
4963
4964         my $ci = ($conf->{cloudinit} //= {});
4965
4966         my $recorded = $ci->{$opt};
4967         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4968
4969         if (defined($new)) {
4970             if (defined($old)) {
4971                 # an existing value is being modified
4972                 if (defined($recorded)) {
4973                     # the value was already not in sync
4974                     if ($new eq $recorded) {
4975                         # a value is being reverted to the cloud-init state:
4976                         delete $ci->{$opt};
4977                         delete $added{$opt};
4978                     } else {
4979                         # the value was changed multiple times, do nothing
4980                     }
4981                 } elsif ($added{$opt}) {
4982                     # the value had been marked as added and is being changed, do nothing
4983                 } else {
4984                     # the value is new, record it:
4985                     $ci->{$opt} = $old;
4986                 }
4987             } else {
4988                 # a new value is being added
4989                 if (defined($recorded)) {
4990                     # it was already not in sync
4991                     if ($new eq $recorded) {
4992                         # a value is being reverted to the cloud-init state:
4993                         delete $ci->{$opt};
4994                         delete $added{$opt};
4995                     } else {
4996                         # the value had temporarily been removed, do nothing
4997                     }
4998                 } elsif ($added{$opt}) {
4999                     # the value had been marked as added already, do nothing
5000                 } else {
5001                     # the value is new, add it
5002                     $added{$opt} = 1;
5003                 }
5004             }
5005         } elsif (!defined($old)) {
5006             # a non-existent value is being removed? ignore...
5007         } else {
5008             # a value is being deleted
5009             if (defined($recorded)) {
5010                 # a value was already recorded, just keep it
5011             } elsif ($added{$opt}) {
5012                 # the value was marked as added, remove it
5013                 delete $added{$opt};
5014             } else {
5015                 # a previously unrecorded value is being removed, record the old value:
5016                 $ci->{$opt} = $old;
5017             }
5018         }
5019
5020         my $added = join(',', sort keys %added);
5021         $ci->{added} = $added if length($added);
5022     };
5023
5024     my $changes = 0;
5025     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5026         if ($fast_plug_option->{$opt}) {
5027             my $new = delete $conf->{pending}->{$opt};
5028             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
5029             $conf->{$opt} = $new;
5030             $changes = 1;
5031         }
5032     }
5033
5034     if ($changes) {
5035         PVE::QemuConfig->write_config($vmid, $conf);
5036     }
5037
5038     my $ostype = $conf->{ostype};
5039     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5040     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5041     my $usb_hotplug = $hotplug_features->{usb}
5042         && min_version($version, 7, 1)
5043         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5044
5045     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5046     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5047
5048     foreach my $opt (sort keys %$pending_delete_hash) {
5049         next if $selection && !$selection->{$opt};
5050         my $force = $pending_delete_hash->{$opt}->{force};
5051         eval {
5052             if ($opt eq 'hotplug') {
5053                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5054             } elsif ($opt eq 'tablet') {
5055                 die "skip\n" if !$hotplug_features->{usb};
5056                 if ($defaults->{tablet}) {
5057                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5058                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5059                         if $arch eq 'aarch64';
5060                 } else {
5061                     vm_deviceunplug($vmid, $conf, 'tablet');
5062                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5063                 }
5064             } elsif ($opt =~ m/^usb(\d+)$/) {
5065                 my $index = $1;
5066                 die "skip\n" if !$usb_hotplug;
5067                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5068                 vm_deviceunplug($vmid, $conf, $opt);
5069             } elsif ($opt eq 'vcpus') {
5070                 die "skip\n" if !$hotplug_features->{cpu};
5071                 qemu_cpu_hotplug($vmid, $conf, undef);
5072             } elsif ($opt eq 'balloon') {
5073                 # enable balloon device is not hotpluggable
5074                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5075                 # here we reset the ballooning value to memory
5076                 my $balloon = $conf->{memory} || $defaults->{memory};
5077                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5078             } elsif ($fast_plug_option->{$opt}) {
5079                 # do nothing
5080             } elsif ($opt =~ m/^net(\d+)$/) {
5081                 die "skip\n" if !$hotplug_features->{network};
5082                 vm_deviceunplug($vmid, $conf, $opt);
5083             } elsif (is_valid_drivename($opt)) {
5084                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5085                 vm_deviceunplug($vmid, $conf, $opt);
5086                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5087             } elsif ($opt =~ m/^memory$/) {
5088                 die "skip\n" if !$hotplug_features->{memory};
5089                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5090             } elsif ($opt eq 'cpuunits') {
5091                 $cgroup->change_cpu_shares(undef);
5092             } elsif ($opt eq 'cpulimit') {
5093                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5094             } else {
5095                 die "skip\n";
5096             }
5097         };
5098         if (my $err = $@) {
5099             &$add_error($opt, $err) if $err ne "skip\n";
5100         } else {
5101             my $old = delete $conf->{$opt};
5102             $cloudinit_record_changed->($conf, $opt, $old, undef);
5103             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5104         }
5105     }
5106
5107     my $cloudinit_opt;
5108     foreach my $opt (keys %{$conf->{pending}}) {
5109         next if $selection && !$selection->{$opt};
5110         my $value = $conf->{pending}->{$opt};
5111         eval {
5112             if ($opt eq 'hotplug') {
5113                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5114             } elsif ($opt eq 'tablet') {
5115                 die "skip\n" if !$hotplug_features->{usb};
5116                 if ($value == 1) {
5117                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5118                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5119                         if $arch eq 'aarch64';
5120                 } elsif ($value == 0) {
5121                     vm_deviceunplug($vmid, $conf, 'tablet');
5122                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5123                 }
5124             } elsif ($opt =~ m/^usb(\d+)$/) {
5125                 my $index = $1;
5126                 die "skip\n" if !$usb_hotplug;
5127                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5128                 my $id = $opt;
5129                 if ($d->{host} =~ m/^spice$/i)  {
5130                     $id = "usbredirdev$index";
5131                 }
5132                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5133             } elsif ($opt eq 'vcpus') {
5134                 die "skip\n" if !$hotplug_features->{cpu};
5135                 qemu_cpu_hotplug($vmid, $conf, $value);
5136             } elsif ($opt eq 'balloon') {
5137                 # enable/disable balloning device is not hotpluggable
5138                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5139                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5140                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5141
5142                 # allow manual ballooning if shares is set to zero
5143                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5144                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5145                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5146                 }
5147             } elsif ($opt =~ m/^net(\d+)$/) {
5148                 # some changes can be done without hotplug
5149                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5150                                     $vmid, $opt, $value, $arch, $machine_type);
5151             } elsif (is_valid_drivename($opt)) {
5152                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5153                 # some changes can be done without hotplug
5154                 my $drive = parse_drive($opt, $value);
5155                 if (drive_is_cloudinit($drive)) {
5156                     $cloudinit_opt = [$opt, $drive];
5157                     # apply all the other changes first, then generate the cloudinit disk
5158                     die "skip\n";
5159                 }
5160                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5161                                      $vmid, $opt, $value, $arch, $machine_type);
5162             } elsif ($opt =~ m/^memory$/) { #dimms
5163                 die "skip\n" if !$hotplug_features->{memory};
5164                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5165             } elsif ($opt eq 'cpuunits') {
5166                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5167                 $cgroup->change_cpu_shares($new_cpuunits);
5168             } elsif ($opt eq 'cpulimit') {
5169                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5170                 $cgroup->change_cpu_quota($cpulimit, 100000);
5171             } elsif ($opt eq 'agent') {
5172                 vmconfig_update_agent($conf, $opt, $value);
5173             } else {
5174                 die "skip\n";  # skip non-hot-pluggable options
5175             }
5176         };
5177         if (my $err = $@) {
5178             &$add_error($opt, $err) if $err ne "skip\n";
5179         } else {
5180             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5181             $conf->{$opt} = $value;
5182             delete $conf->{pending}->{$opt};
5183         }
5184     }
5185
5186     if (defined($cloudinit_opt)) {
5187         my ($opt, $drive) = @$cloudinit_opt;
5188         my $value = $conf->{pending}->{$opt};
5189         eval {
5190             my $temp = {%$conf, $opt => $value};
5191             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5192             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5193                                  $vmid, $opt, $value, $arch, $machine_type);
5194         };
5195         if (my $err = $@) {
5196             &$add_error($opt, $err) if $err ne "skip\n";
5197         } else {
5198             $conf->{$opt} = $value;
5199             delete $conf->{pending}->{$opt};
5200         }
5201     }
5202
5203     # unplug xhci controller if no usb device is left
5204     if ($usb_hotplug) {
5205         my $has_usb = 0;
5206         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5207             next if !defined($conf->{"usb$i"});
5208             $has_usb = 1;
5209             last;
5210         }
5211         if (!$has_usb) {
5212             vm_deviceunplug($vmid, $conf, 'xhci');
5213         }
5214     }
5215
5216     PVE::QemuConfig->write_config($vmid, $conf);
5217
5218     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5219         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5220     }
5221 }
5222
5223 sub try_deallocate_drive {
5224     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5225
5226     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5227         my $volid = $drive->{file};
5228         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5229             my $sid = PVE::Storage::parse_volume_id($volid);
5230             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5231
5232             # check if the disk is really unused
5233             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5234                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5235             PVE::Storage::vdisk_free($storecfg, $volid);
5236             return 1;
5237         } else {
5238             # If vm is not owner of this disk remove from config
5239             return 1;
5240         }
5241     }
5242
5243     return;
5244 }
5245
5246 sub vmconfig_delete_or_detach_drive {
5247     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5248
5249     my $drive = parse_drive($opt, $conf->{$opt});
5250
5251     my $rpcenv = PVE::RPCEnvironment::get();
5252     my $authuser = $rpcenv->get_user();
5253
5254     if ($force) {
5255         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5256         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5257     } else {
5258         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5259     }
5260 }
5261
5262
5263
5264 sub vmconfig_apply_pending {
5265     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5266
5267     return if !scalar(keys %{$conf->{pending}});
5268
5269     my $add_apply_error = sub {
5270         my ($opt, $msg) = @_;
5271         my $err_msg = "unable to apply pending change $opt : $msg";
5272         $errors->{$opt} = $err_msg;
5273         warn $err_msg;
5274     };
5275
5276     # cold plug
5277
5278     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5279     foreach my $opt (sort keys %$pending_delete_hash) {
5280         my $force = $pending_delete_hash->{$opt}->{force};
5281         eval {
5282             if ($opt =~ m/^unused/) {
5283                 die "internal error";
5284             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5285                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5286             }
5287         };
5288         if (my $err = $@) {
5289             $add_apply_error->($opt, $err);
5290         } else {
5291             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5292             delete $conf->{$opt};
5293         }
5294     }
5295
5296     PVE::QemuConfig->cleanup_pending($conf);
5297
5298     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5299
5300     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5301         next if $opt eq 'delete'; # just to be sure
5302         eval {
5303             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5304                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5305             }
5306         };
5307         if (my $err = $@) {
5308             $add_apply_error->($opt, $err);
5309         } else {
5310
5311             if (is_valid_drivename($opt)) {
5312                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5313                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5314             }
5315
5316             $conf->{$opt} = delete $conf->{pending}->{$opt};
5317         }
5318     }
5319
5320     # write all changes at once to avoid unnecessary i/o
5321     PVE::QemuConfig->write_config($vmid, $conf);
5322     if ($generate_cloudinit) {
5323         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5324             # After successful generation and if there were changes to be applied, update the
5325             # config to drop the {cloudinit} entry.
5326             PVE::QemuConfig->write_config($vmid, $conf);
5327         }
5328     }
5329 }
5330
5331 sub vmconfig_update_net {
5332     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5333
5334     my $newnet = parse_net($value);
5335
5336     if ($conf->{$opt}) {
5337         my $oldnet = parse_net($conf->{$opt});
5338
5339         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5340             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5341             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5342             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5343             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5344
5345             # for non online change, we try to hot-unplug
5346             die "skip\n" if !$hotplug;
5347             vm_deviceunplug($vmid, $conf, $opt);
5348         } else {
5349
5350             die "internal error" if $opt !~ m/net(\d+)/;
5351             my $iface = "tap${vmid}i$1";
5352
5353             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5354                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5355                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5356                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5357                 PVE::Network::tap_unplug($iface);
5358
5359                 if ($have_sdn) {
5360                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5361                 } else {
5362                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5363                 }
5364             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5365                 # Rate can be applied on its own but any change above needs to
5366                 # include the rate in tap_plug since OVS resets everything.
5367                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5368             }
5369
5370             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5371                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5372             }
5373
5374             return 1;
5375         }
5376     }
5377
5378     if ($hotplug) {
5379         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5380     } else {
5381         die "skip\n";
5382     }
5383 }
5384
5385 sub vmconfig_update_agent {
5386     my ($conf, $opt, $value) = @_;
5387
5388     die "skip\n" if !$conf->{$opt};
5389
5390     my $hotplug_options = { fstrim_cloned_disks => 1 };
5391
5392     my $old_agent = parse_guest_agent($conf);
5393     my $agent = parse_guest_agent({$opt => $value});
5394
5395     for my $option (keys %$agent) { # added/changed options
5396         next if defined($hotplug_options->{$option});
5397         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5398     }
5399
5400     for my $option (keys %$old_agent) { # removed options
5401         next if defined($hotplug_options->{$option});
5402         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5403     }
5404
5405     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5406 }
5407
5408 sub vmconfig_update_disk {
5409     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5410
5411     my $drive = parse_drive($opt, $value);
5412
5413     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5414         my $media = $drive->{media} || 'disk';
5415         my $oldmedia = $old_drive->{media} || 'disk';
5416         die "unable to change media type\n" if $media ne $oldmedia;
5417
5418         if (!drive_is_cdrom($old_drive)) {
5419
5420             if ($drive->{file} ne $old_drive->{file}) {
5421
5422                 die "skip\n" if !$hotplug;
5423
5424                 # unplug and register as unused
5425                 vm_deviceunplug($vmid, $conf, $opt);
5426                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5427
5428             } else {
5429                 # update existing disk
5430
5431                 # skip non hotpluggable value
5432                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5433                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5434                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5435                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5436                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5437                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5438                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5439                     die "skip\n";
5440                 }
5441
5442                 # apply throttle
5443                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5444                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5445                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5446                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5447                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5448                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5449                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5450                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5451                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5452                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5453                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5454                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5455                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5456                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5457                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5458                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5459                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5460                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5461
5462                     qemu_block_set_io_throttle(
5463                         $vmid,"drive-$opt",
5464                         ($drive->{mbps} || 0)*1024*1024,
5465                         ($drive->{mbps_rd} || 0)*1024*1024,
5466                         ($drive->{mbps_wr} || 0)*1024*1024,
5467                          $drive->{iops} || 0,
5468                          $drive->{iops_rd} || 0,
5469                          $drive->{iops_wr} || 0,
5470                         ($drive->{mbps_max} || 0)*1024*1024,
5471                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5472                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5473                          $drive->{iops_max} || 0,
5474                          $drive->{iops_rd_max} || 0,
5475                          $drive->{iops_wr_max} || 0,
5476                          $drive->{bps_max_length} || 1,
5477                          $drive->{bps_rd_max_length} || 1,
5478                          $drive->{bps_wr_max_length} || 1,
5479                          $drive->{iops_max_length} || 1,
5480                          $drive->{iops_rd_max_length} || 1,
5481                          $drive->{iops_wr_max_length} || 1,
5482                     );
5483
5484                 }
5485
5486                 return 1;
5487             }
5488
5489         } else { # cdrom
5490
5491             if ($drive->{file} eq 'none') {
5492                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5493                 if (drive_is_cloudinit($old_drive)) {
5494                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5495                 }
5496             } else {
5497                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5498
5499                 # force eject if locked
5500                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5501
5502                 if ($path) {
5503                     mon_cmd($vmid, "blockdev-change-medium",
5504                         id => "$opt", filename => "$path");
5505                 }
5506             }
5507
5508             return 1;
5509         }
5510     }
5511
5512     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5513     # hotplug new disks
5514     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5515     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5516 }
5517
5518 sub vmconfig_update_cloudinit_drive {
5519     my ($storecfg, $conf, $vmid) = @_;
5520
5521     my $cloudinit_ds = undef;
5522     my $cloudinit_drive = undef;
5523
5524     PVE::QemuConfig->foreach_volume($conf, sub {
5525         my ($ds, $drive) = @_;
5526         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5527             $cloudinit_ds = $ds;
5528             $cloudinit_drive = $drive;
5529         }
5530     });
5531
5532     return if !$cloudinit_drive;
5533
5534     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5535         PVE::QemuConfig->write_config($vmid, $conf);
5536     }
5537
5538     my $running = PVE::QemuServer::check_running($vmid);
5539
5540     if ($running) {
5541         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5542         if ($path) {
5543             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5544             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5545         }
5546     }
5547 }
5548
5549 # called in locked context by incoming migration
5550 sub vm_migrate_get_nbd_disks {
5551     my ($storecfg, $conf, $replicated_volumes) = @_;
5552
5553     my $local_volumes = {};
5554     PVE::QemuConfig->foreach_volume($conf, sub {
5555         my ($ds, $drive) = @_;
5556
5557         return if drive_is_cdrom($drive);
5558         return if $ds eq 'tpmstate0';
5559
5560         my $volid = $drive->{file};
5561
5562         return if !$volid;
5563
5564         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5565
5566         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5567         return if $scfg->{shared};
5568
5569         my $format = qemu_img_format($scfg, $volname);
5570
5571         # replicated disks re-use existing state via bitmap
5572         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5573         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5574     });
5575     return $local_volumes;
5576 }
5577
5578 # called in locked context by incoming migration
5579 sub vm_migrate_alloc_nbd_disks {
5580     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5581
5582     my $nbd = {};
5583     foreach my $opt (sort keys %$source_volumes) {
5584         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5585
5586         if ($use_existing) {
5587             $nbd->{$opt}->{drivestr} = print_drive($drive);
5588             $nbd->{$opt}->{volid} = $volid;
5589             $nbd->{$opt}->{replicated} = 1;
5590             next;
5591         }
5592
5593         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5594
5595         # order of precedence, filtered by whether storage supports it:
5596         # 1. explicit requested format
5597         # 2. default format of storage
5598         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5599         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5600
5601         my $size = $drive->{size} / 1024;
5602         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5603         my $newdrive = $drive;
5604         $newdrive->{format} = $format;
5605         $newdrive->{file} = $newvolid;
5606         my $drivestr = print_drive($newdrive);
5607         $nbd->{$opt}->{drivestr} = $drivestr;
5608         $nbd->{$opt}->{volid} = $newvolid;
5609     }
5610
5611     return $nbd;
5612 }
5613
5614 # see vm_start_nolock for parameters, additionally:
5615 # migrate_opts:
5616 #   storagemap = parsed storage map for allocating NBD disks
5617 sub vm_start {
5618     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5619
5620     return PVE::QemuConfig->lock_config($vmid, sub {
5621         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5622
5623         die "you can't start a vm if it's a template\n"
5624             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5625
5626         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5627         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5628
5629         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5630
5631         if ($has_backup_lock && $running) {
5632             # a backup is currently running, attempt to start the guest in the
5633             # existing QEMU instance
5634             return vm_resume($vmid);
5635         }
5636
5637         PVE::QemuConfig->check_lock($conf)
5638             if !($params->{skiplock} || $has_suspended_lock);
5639
5640         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5641
5642         die "VM $vmid already running\n" if $running;
5643
5644         if (my $storagemap = $migrate_opts->{storagemap}) {
5645             my $replicated = $migrate_opts->{replicated_volumes};
5646             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5647             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5648
5649             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5650                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5651             }
5652         }
5653
5654         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5655     });
5656 }
5657
5658
5659 # params:
5660 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5661 #   skiplock => 0/1, skip checking for config lock
5662 #   skiptemplate => 0/1, skip checking whether VM is template
5663 #   forcemachine => to force QEMU machine (rollback/migration)
5664 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5665 #   timeout => in seconds
5666 #   paused => start VM in paused state (backup)
5667 #   resume => resume from hibernation
5668 #   pbs-backing => {
5669 #      sata0 => {
5670 #         repository
5671 #         snapshot
5672 #         keyfile
5673 #         archive
5674 #      },
5675 #      virtio2 => ...
5676 #   }
5677 # migrate_opts:
5678 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5679 #   migratedfrom => source node
5680 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5681 #   network => CIDR of migration network
5682 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5683 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5684 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5685 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5686 #       contained in config
5687 sub vm_start_nolock {
5688     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5689
5690     my $statefile = $params->{statefile};
5691     my $resume = $params->{resume};
5692
5693     my $migratedfrom = $migrate_opts->{migratedfrom};
5694     my $migration_type = $migrate_opts->{type};
5695
5696     my $res = {};
5697
5698     # clean up leftover reboot request files
5699     eval { clear_reboot_request($vmid); };
5700     warn $@ if $@;
5701
5702     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5703         vmconfig_apply_pending($vmid, $conf, $storecfg);
5704         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5705     }
5706
5707     # don't regenerate the ISO if the VM is started as part of a live migration
5708     # this way we can reuse the old ISO with the correct config
5709     if (!$migratedfrom) {
5710         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5711             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5712             # $conf->{cloudinit}, so we could just not do this?
5713             # But we do it above, so for now let's be consistent.
5714             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5715         }
5716     }
5717
5718     # override offline migrated volumes, conf is out of date still
5719     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5720         for my $key (sort keys $offline_volumes->%*) {
5721             my $parsed = parse_drive($key, $conf->{$key});
5722             $parsed->{file} = $offline_volumes->{$key};
5723             $conf->{$key} = print_drive($parsed);
5724         }
5725     }
5726
5727     my $defaults = load_defaults();
5728
5729     # set environment variable useful inside network script
5730     # for remote migration the config is available on the target node!
5731     if (!$migrate_opts->{remote_node}) {
5732         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5733     }
5734
5735     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5736
5737     my $forcemachine = $params->{forcemachine};
5738     my $forcecpu = $params->{forcecpu};
5739     if ($resume) {
5740         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5741         $forcemachine = $conf->{runningmachine};
5742         $forcecpu = $conf->{runningcpu};
5743         print "Resuming suspended VM\n";
5744     }
5745
5746     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5747         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5748
5749     my $migration_ip;
5750     my $get_migration_ip = sub {
5751         my ($nodename) = @_;
5752
5753         return $migration_ip if defined($migration_ip);
5754
5755         my $cidr = $migrate_opts->{network};
5756
5757         if (!defined($cidr)) {
5758             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5759             $cidr = $dc_conf->{migration}->{network};
5760         }
5761
5762         if (defined($cidr)) {
5763             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5764
5765             die "could not get IP: no address configured on local " .
5766                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5767
5768             die "could not get IP: multiple addresses configured on local " .
5769                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5770
5771             $migration_ip = @$ips[0];
5772         }
5773
5774         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5775             if !defined($migration_ip);
5776
5777         return $migration_ip;
5778     };
5779
5780     if ($statefile) {
5781         if ($statefile eq 'tcp') {
5782             my $migrate = $res->{migrate} = { proto => 'tcp' };
5783             $migrate->{addr} = "localhost";
5784             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5785             my $nodename = nodename();
5786
5787             if (!defined($migration_type)) {
5788                 if (defined($datacenterconf->{migration}->{type})) {
5789                     $migration_type = $datacenterconf->{migration}->{type};
5790                 } else {
5791                     $migration_type = 'secure';
5792                 }
5793             }
5794
5795             if ($migration_type eq 'insecure') {
5796                 $migrate->{addr} = $get_migration_ip->($nodename);
5797                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5798             }
5799
5800             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5801             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5802             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5803             push @$cmd, '-incoming', $migrate->{uri};
5804             push @$cmd, '-S';
5805
5806         } elsif ($statefile eq 'unix') {
5807             # should be default for secure migrations as a ssh TCP forward
5808             # tunnel is not deterministic reliable ready and fails regurarly
5809             # to set up in time, so use UNIX socket forwards
5810             my $migrate = $res->{migrate} = { proto => 'unix' };
5811             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5812             unlink $migrate->{addr};
5813
5814             $migrate->{uri} = "unix:$migrate->{addr}";
5815             push @$cmd, '-incoming', $migrate->{uri};
5816             push @$cmd, '-S';
5817
5818         } elsif (-e $statefile) {
5819             push @$cmd, '-loadstate', $statefile;
5820         } else {
5821             my $statepath = PVE::Storage::path($storecfg, $statefile);
5822             push @$vollist, $statefile;
5823             push @$cmd, '-loadstate', $statepath;
5824         }
5825     } elsif ($params->{paused}) {
5826         push @$cmd, '-S';
5827     }
5828
5829     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5830
5831     my $pci_reserve_list = [];
5832     for my $device (values $pci_devices->%*) {
5833         next if $device->{mdev}; # we don't reserve for mdev devices
5834         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5835     }
5836
5837     # reserve all PCI IDs before actually doing anything with them
5838     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5839
5840     eval {
5841         my $uuid;
5842         for my $id (sort keys %$pci_devices) {
5843             my $d = $pci_devices->{$id};
5844             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5845
5846             my $chosen_mdev;
5847             for my $dev ($d->{ids}->@*) {
5848                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5849                 if ($d->{mdev}) {
5850                     warn $@ if $@;
5851                     $chosen_mdev = $info;
5852                     last if $chosen_mdev; # if successful, we're done
5853                 } else {
5854                     die $@ if $@;
5855                 }
5856             }
5857
5858             next if !$d->{mdev};
5859             die "could not create mediated device\n" if !defined($chosen_mdev);
5860
5861             # nvidia grid needs the uuid of the mdev as qemu parameter
5862             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5863                 if (defined($conf->{smbios1})) {
5864                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5865                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5866                 }
5867                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5868             }
5869         }
5870         push @$cmd, '-uuid', $uuid if defined($uuid);
5871     };
5872     if (my $err = $@) {
5873         eval { cleanup_pci_devices($vmid, $conf) };
5874         warn $@ if $@;
5875         die $err;
5876     }
5877
5878     PVE::Storage::activate_volumes($storecfg, $vollist);
5879
5880
5881     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5882     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5883     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5884     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5885     # timeout should be more than enough here...
5886     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5887
5888     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5889
5890     my %run_params = (
5891         timeout => $statefile ? undef : $start_timeout,
5892         umask => 0077,
5893         noerr => 1,
5894     );
5895
5896     # when migrating, prefix QEMU output so other side can pick up any
5897     # errors that might occur and show the user
5898     if ($migratedfrom) {
5899         $run_params{quiet} = 1;
5900         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5901     }
5902
5903     my %systemd_properties = (
5904         Slice => 'qemu.slice',
5905         KillMode => 'process',
5906         SendSIGKILL => 0,
5907         TimeoutStopUSec => ULONG_MAX, # infinity
5908     );
5909
5910     if (PVE::CGroup::cgroup_mode() == 2) {
5911         $systemd_properties{CPUWeight} = $cpuunits;
5912     } else {
5913         $systemd_properties{CPUShares} = $cpuunits;
5914     }
5915
5916     if (my $cpulimit = $conf->{cpulimit}) {
5917         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5918     }
5919     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5920
5921     my $run_qemu = sub {
5922         PVE::Tools::run_fork sub {
5923             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5924
5925             my $tpmpid;
5926             if (my $tpm = $conf->{tpmstate0}) {
5927                 # start the TPM emulator so QEMU can connect on start
5928                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5929             }
5930
5931             my $exitcode = run_command($cmd, %run_params);
5932             if ($exitcode) {
5933                 if ($tpmpid) {
5934                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5935                     kill 'TERM', $tpmpid;
5936                 }
5937                 die "QEMU exited with code $exitcode\n";
5938             }
5939         };
5940     };
5941
5942     if ($conf->{hugepages}) {
5943
5944         my $code = sub {
5945             my $hotplug_features =
5946                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5947             my $hugepages_topology =
5948                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5949
5950             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5951
5952             PVE::QemuServer::Memory::hugepages_mount();
5953             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5954
5955             eval { $run_qemu->() };
5956             if (my $err = $@) {
5957                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5958                     if !$conf->{keephugepages};
5959                 die $err;
5960             }
5961
5962             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5963                 if !$conf->{keephugepages};
5964         };
5965         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5966
5967     } else {
5968         eval { $run_qemu->() };
5969     }
5970
5971     if (my $err = $@) {
5972         # deactivate volumes if start fails
5973         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5974         warn $@ if $@;
5975         eval { cleanup_pci_devices($vmid, $conf) };
5976         warn $@ if $@;
5977
5978         die "start failed: $err";
5979     }
5980
5981     # re-reserve all PCI IDs now that we can know the actual VM PID
5982     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5983     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5984     warn $@ if $@;
5985
5986     if (defined($res->{migrate})) {
5987         print "migration listens on $res->{migrate}->{uri}\n";
5988     } elsif ($statefile) {
5989         eval { mon_cmd($vmid, "cont"); };
5990         warn $@ if $@;
5991     }
5992
5993     #start nbd server for storage migration
5994     if (my $nbd = $migrate_opts->{nbd}) {
5995         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5996
5997         my $migrate_storage_uri;
5998         # nbd_protocol_version > 0 for unix socket support
5999         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
6000             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
6001             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
6002             $migrate_storage_uri = "nbd:unix:$socket_path";
6003             $res->{migrate}->{unix_sockets} = [$socket_path];
6004         } else {
6005             my $nodename = nodename();
6006             my $localip = $get_migration_ip->($nodename);
6007             my $pfamily = PVE::Tools::get_host_address_family($nodename);
6008             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
6009
6010             mon_cmd($vmid, "nbd-server-start", addr => {
6011                 type => 'inet',
6012                 data => {
6013                     host => "${localip}",
6014                     port => "${storage_migrate_port}",
6015                 },
6016             });
6017             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
6018             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
6019         }
6020
6021         my $block_info = mon_cmd($vmid, "query-block");
6022         $block_info = { map { $_->{device} => $_ } $block_info->@* };
6023
6024         foreach my $opt (sort keys %$nbd) {
6025             my $drivestr = $nbd->{$opt}->{drivestr};
6026             my $volid = $nbd->{$opt}->{volid};
6027
6028             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
6029
6030             mon_cmd(
6031                 $vmid,
6032                 "block-export-add",
6033                 id => "drive-$opt",
6034                 'node-name' => $block_node,
6035                 writable => JSON::true,
6036                 type => "nbd",
6037                 name => "drive-$opt", # NBD export name
6038             );
6039
6040             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6041             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6042             print "re-using replicated volume: $opt - $volid\n"
6043                 if $nbd->{$opt}->{replicated};
6044
6045             $res->{drives}->{$opt} = $nbd->{$opt};
6046             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6047         }
6048     }
6049
6050     if ($migratedfrom) {
6051         eval {
6052             set_migration_caps($vmid);
6053         };
6054         warn $@ if $@;
6055
6056         if ($spice_port) {
6057             print "spice listens on port $spice_port\n";
6058             $res->{spice_port} = $spice_port;
6059             if ($migrate_opts->{spice_ticket}) {
6060                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6061                     $migrate_opts->{spice_ticket});
6062                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6063             }
6064         }
6065
6066     } else {
6067         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6068             if !$statefile && $conf->{balloon};
6069
6070         foreach my $opt (keys %$conf) {
6071             next if $opt !~  m/^net\d+$/;
6072             my $nicconf = parse_net($conf->{$opt});
6073             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6074         }
6075         add_nets_bridge_fdb($conf, $vmid);
6076     }
6077
6078    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6079         eval {
6080             mon_cmd(
6081                 $vmid,
6082                 'qom-set',
6083                 path => "machine/peripheral/balloon0",
6084                 property => "guest-stats-polling-interval",
6085                 value => 2
6086             );
6087         };
6088         log_warn("could not set polling interval for ballooning - $@") if $@;
6089     }
6090
6091     if ($resume) {
6092         print "Resumed VM, removing state\n";
6093         if (my $vmstate = $conf->{vmstate}) {
6094             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6095             PVE::Storage::vdisk_free($storecfg, $vmstate);
6096         }
6097         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6098         PVE::QemuConfig->write_config($vmid, $conf);
6099     }
6100
6101     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6102
6103     return $res;
6104 }
6105
6106 sub vm_commandline {
6107     my ($storecfg, $vmid, $snapname) = @_;
6108
6109     my $conf = PVE::QemuConfig->load_config($vmid);
6110
6111     my ($forcemachine, $forcecpu);
6112     if ($snapname) {
6113         my $snapshot = $conf->{snapshots}->{$snapname};
6114         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6115
6116         # check for machine or CPU overrides in snapshot
6117         $forcemachine = $snapshot->{runningmachine};
6118         $forcecpu = $snapshot->{runningcpu};
6119
6120         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6121
6122         $conf = $snapshot;
6123     }
6124
6125     my $defaults = load_defaults();
6126
6127     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6128
6129     return PVE::Tools::cmd2string($cmd);
6130 }
6131
6132 sub vm_reset {
6133     my ($vmid, $skiplock) = @_;
6134
6135     PVE::QemuConfig->lock_config($vmid, sub {
6136
6137         my $conf = PVE::QemuConfig->load_config($vmid);
6138
6139         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6140
6141         mon_cmd($vmid, "system_reset");
6142     });
6143 }
6144
6145 sub get_vm_volumes {
6146     my ($conf) = @_;
6147
6148     my $vollist = [];
6149     foreach_volid($conf, sub {
6150         my ($volid, $attr) = @_;
6151
6152         return if $volid =~ m|^/|;
6153
6154         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6155         return if !$sid;
6156
6157         push @$vollist, $volid;
6158     });
6159
6160     return $vollist;
6161 }
6162
6163 sub cleanup_pci_devices {
6164     my ($vmid, $conf) = @_;
6165
6166     foreach my $key (keys %$conf) {
6167         next if $key !~ m/^hostpci(\d+)$/;
6168         my $hostpciindex = $1;
6169         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6170         my $d = parse_hostpci($conf->{$key});
6171         if ($d->{mdev}) {
6172             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6173             # don't want to break ABI just for this two liner
6174             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6175
6176             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6177             # out when we do it first. so wait for 10 seconds and then try it
6178             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/) {
6179                 sleep 10;
6180             }
6181
6182             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6183         }
6184     }
6185     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6186 }
6187
6188 sub vm_stop_cleanup {
6189     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6190
6191     eval {
6192
6193         if (!$keepActive) {
6194             my $vollist = get_vm_volumes($conf);
6195             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6196
6197             if (my $tpmdrive = $conf->{tpmstate0}) {
6198                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6199                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6200                 if ($storeid) {
6201                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6202                 }
6203             }
6204         }
6205
6206         foreach my $ext (qw(mon qmp pid vnc qga)) {
6207             unlink "/var/run/qemu-server/${vmid}.$ext";
6208         }
6209
6210         if ($conf->{ivshmem}) {
6211             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6212             # just delete it for now, VMs which have this already open do not
6213             # are affected, but new VMs will get a separated one. If this
6214             # becomes an issue we either add some sort of ref-counting or just
6215             # add a "don't delete on stop" flag to the ivshmem format.
6216             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6217         }
6218
6219         cleanup_pci_devices($vmid, $conf);
6220
6221         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6222     };
6223     warn $@ if $@; # avoid errors - just warn
6224 }
6225
6226 # call only in locked context
6227 sub _do_vm_stop {
6228     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6229
6230     my $pid = check_running($vmid, $nocheck);
6231     return if !$pid;
6232
6233     my $conf;
6234     if (!$nocheck) {
6235         $conf = PVE::QemuConfig->load_config($vmid);
6236         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6237         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6238             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6239             $timeout = $opts->{down} if $opts->{down};
6240         }
6241         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6242     }
6243
6244     eval {
6245         if ($shutdown) {
6246             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6247                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6248             } else {
6249                 mon_cmd($vmid, "system_powerdown");
6250             }
6251         } else {
6252             mon_cmd($vmid, "quit");
6253         }
6254     };
6255     my $err = $@;
6256
6257     if (!$err) {
6258         $timeout = 60 if !defined($timeout);
6259
6260         my $count = 0;
6261         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6262             $count++;
6263             sleep 1;
6264         }
6265
6266         if ($count >= $timeout) {
6267             if ($force) {
6268                 warn "VM still running - terminating now with SIGTERM\n";
6269                 kill 15, $pid;
6270             } else {
6271                 die "VM quit/powerdown failed - got timeout\n";
6272             }
6273         } else {
6274             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6275             return;
6276         }
6277     } else {
6278        if (!check_running($vmid, $nocheck)) {
6279             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6280             return;
6281        }
6282        if ($force) {
6283             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6284             kill 15, $pid;
6285         } else {
6286             die "VM quit/powerdown failed\n";
6287         }
6288     }
6289
6290     # wait again
6291     $timeout = 10;
6292
6293     my $count = 0;
6294     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6295         $count++;
6296         sleep 1;
6297     }
6298
6299     if ($count >= $timeout) {
6300         warn "VM still running - terminating now with SIGKILL\n";
6301         kill 9, $pid;
6302         sleep 1;
6303     }
6304
6305     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6306 }
6307
6308 # Note: use $nocheck to skip tests if VM configuration file exists.
6309 # We need that when migration VMs to other nodes (files already moved)
6310 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6311 sub vm_stop {
6312     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6313
6314     $force = 1 if !defined($force) && !$shutdown;
6315
6316     if ($migratedfrom){
6317         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6318         kill 15, $pid if $pid;
6319         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6320         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6321         return;
6322     }
6323
6324     PVE::QemuConfig->lock_config($vmid, sub {
6325         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6326    });
6327 }
6328
6329 sub vm_reboot {
6330     my ($vmid, $timeout) = @_;
6331
6332     PVE::QemuConfig->lock_config($vmid, sub {
6333         eval {
6334
6335             # only reboot if running, as qmeventd starts it again on a stop event
6336             return if !check_running($vmid);
6337
6338             create_reboot_request($vmid);
6339
6340             my $storecfg = PVE::Storage::config();
6341             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6342
6343         };
6344         if (my $err = $@) {
6345             # avoid that the next normal shutdown will be confused for a reboot
6346             clear_reboot_request($vmid);
6347             die $err;
6348         }
6349    });
6350 }
6351
6352 # note: if using the statestorage parameter, the caller has to check privileges
6353 sub vm_suspend {
6354     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6355
6356     my $conf;
6357     my $path;
6358     my $storecfg;
6359     my $vmstate;
6360
6361     PVE::QemuConfig->lock_config($vmid, sub {
6362
6363         $conf = PVE::QemuConfig->load_config($vmid);
6364
6365         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6366         PVE::QemuConfig->check_lock($conf)
6367             if !($skiplock || $is_backing_up);
6368
6369         die "cannot suspend to disk during backup\n"
6370             if $is_backing_up && $includestate;
6371
6372         if ($includestate) {
6373             $conf->{lock} = 'suspending';
6374             my $date = strftime("%Y-%m-%d", localtime(time()));
6375             $storecfg = PVE::Storage::config();
6376             if (!$statestorage) {
6377                 $statestorage = find_vmstate_storage($conf, $storecfg);
6378                 # check permissions for the storage
6379                 my $rpcenv = PVE::RPCEnvironment::get();
6380                 if ($rpcenv->{type} ne 'cli') {
6381                     my $authuser = $rpcenv->get_user();
6382                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6383                 }
6384             }
6385
6386
6387             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6388                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6389             $path = PVE::Storage::path($storecfg, $vmstate);
6390             PVE::QemuConfig->write_config($vmid, $conf);
6391         } else {
6392             mon_cmd($vmid, "stop");
6393         }
6394     });
6395
6396     if ($includestate) {
6397         # save vm state
6398         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6399
6400         eval {
6401             set_migration_caps($vmid, 1);
6402             mon_cmd($vmid, "savevm-start", statefile => $path);
6403             for(;;) {
6404                 my $state = mon_cmd($vmid, "query-savevm");
6405                 if (!$state->{status}) {
6406                     die "savevm not active\n";
6407                 } elsif ($state->{status} eq 'active') {
6408                     sleep(1);
6409                     next;
6410                 } elsif ($state->{status} eq 'completed') {
6411                     print "State saved, quitting\n";
6412                     last;
6413                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6414                     die "query-savevm failed with error '$state->{error}'\n"
6415                 } else {
6416                     die "query-savevm returned status '$state->{status}'\n";
6417                 }
6418             }
6419         };
6420         my $err = $@;
6421
6422         PVE::QemuConfig->lock_config($vmid, sub {
6423             $conf = PVE::QemuConfig->load_config($vmid);
6424             if ($err) {
6425                 # cleanup, but leave suspending lock, to indicate something went wrong
6426                 eval {
6427                     mon_cmd($vmid, "savevm-end");
6428                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6429                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6430                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6431                     PVE::QemuConfig->write_config($vmid, $conf);
6432                 };
6433                 warn $@ if $@;
6434                 die $err;
6435             }
6436
6437             die "lock changed unexpectedly\n"
6438                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6439
6440             mon_cmd($vmid, "quit");
6441             $conf->{lock} = 'suspended';
6442             PVE::QemuConfig->write_config($vmid, $conf);
6443         });
6444     }
6445 }
6446
6447 # $nocheck is set when called as part of a migration - in this context the
6448 # location of the config file (source or target node) is not deterministic,
6449 # since migration cannot wait for pmxcfs to process the rename
6450 sub vm_resume {
6451     my ($vmid, $skiplock, $nocheck) = @_;
6452
6453     PVE::QemuConfig->lock_config($vmid, sub {
6454         my $res = mon_cmd($vmid, 'query-status');
6455         my $resume_cmd = 'cont';
6456         my $reset = 0;
6457         my $conf;
6458         if ($nocheck) {
6459             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6460             if ($@) {
6461                 my $vmlist = PVE::Cluster::get_vmlist();
6462                 if (exists($vmlist->{ids}->{$vmid})) {
6463                     my $node = $vmlist->{ids}->{$vmid}->{node};
6464                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6465                 }
6466                 if (!$conf) {
6467                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6468                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6469                 }
6470             }
6471         } else {
6472             $conf = PVE::QemuConfig->load_config($vmid);
6473         }
6474
6475         if ($res->{status}) {
6476             return if $res->{status} eq 'running'; # job done, go home
6477             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6478             $reset = 1 if $res->{status} eq 'shutdown';
6479         }
6480
6481         if (!$nocheck) {
6482             PVE::QemuConfig->check_lock($conf)
6483                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6484         }
6485
6486         if ($reset) {
6487             # required if a VM shuts down during a backup and we get a resume
6488             # request before the backup finishes for example
6489             mon_cmd($vmid, "system_reset");
6490         }
6491
6492         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6493
6494         mon_cmd($vmid, $resume_cmd);
6495     });
6496 }
6497
6498 sub vm_sendkey {
6499     my ($vmid, $skiplock, $key) = @_;
6500
6501     PVE::QemuConfig->lock_config($vmid, sub {
6502
6503         my $conf = PVE::QemuConfig->load_config($vmid);
6504
6505         # there is no qmp command, so we use the human monitor command
6506         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6507         die $res if $res ne '';
6508     });
6509 }
6510
6511 sub check_bridge_access {
6512     my ($rpcenv, $authuser, $conf) = @_;
6513
6514     return 1 if $authuser eq 'root@pam';
6515
6516     for my $opt (sort keys $conf->%*) {
6517         next if $opt !~ m/^net\d+$/;
6518         my $net = parse_net($conf->{$opt});
6519         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6520         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6521     }
6522     return 1;
6523 };
6524
6525 sub check_mapping_access {
6526     my ($rpcenv, $user, $conf) = @_;
6527
6528     for my $opt (keys $conf->%*) {
6529         if ($opt =~ m/^usb\d+$/) {
6530             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6531             if (my $host = $device->{host}) {
6532                 die "only root can set '$opt' config for real devices\n"
6533                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6534             } elsif ($device->{mapping}) {
6535                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6536             } else {
6537                 die "either 'host' or 'mapping' must be set.\n";
6538             }
6539         } elsif ($opt =~ m/^hostpci\d+$/) {
6540             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6541             if ($device->{host}) {
6542                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6543             } elsif ($device->{mapping}) {
6544                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6545             } else {
6546                 die "either 'host' or 'mapping' must be set.\n";
6547             }
6548        }
6549    }
6550 };
6551
6552 sub check_restore_permissions {
6553     my ($rpcenv, $user, $conf) = @_;
6554
6555     check_bridge_access($rpcenv, $user, $conf);
6556     check_mapping_access($rpcenv, $user, $conf);
6557 }
6558 # vzdump restore implementaion
6559
6560 sub tar_archive_read_firstfile {
6561     my $archive = shift;
6562
6563     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6564
6565     # try to detect archive type first
6566     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6567         die "unable to open file '$archive'\n";
6568     my $firstfile = <$fh>;
6569     kill 15, $pid;
6570     close $fh;
6571
6572     die "ERROR: archive contaions no data\n" if !$firstfile;
6573     chomp $firstfile;
6574
6575     return $firstfile;
6576 }
6577
6578 sub tar_restore_cleanup {
6579     my ($storecfg, $statfile) = @_;
6580
6581     print STDERR "starting cleanup\n";
6582
6583     if (my $fd = IO::File->new($statfile, "r")) {
6584         while (defined(my $line = <$fd>)) {
6585             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6586                 my $volid = $2;
6587                 eval {
6588                     if ($volid =~ m|^/|) {
6589                         unlink $volid || die 'unlink failed\n';
6590                     } else {
6591                         PVE::Storage::vdisk_free($storecfg, $volid);
6592                     }
6593                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6594                 };
6595                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6596             } else {
6597                 print STDERR "unable to parse line in statfile - $line";
6598             }
6599         }
6600         $fd->close();
6601     }
6602 }
6603
6604 sub restore_file_archive {
6605     my ($archive, $vmid, $user, $opts) = @_;
6606
6607     return restore_vma_archive($archive, $vmid, $user, $opts)
6608         if $archive eq '-';
6609
6610     my $info = PVE::Storage::archive_info($archive);
6611     my $format = $opts->{format} // $info->{format};
6612     my $comp = $info->{compression};
6613
6614     # try to detect archive format
6615     if ($format eq 'tar') {
6616         return restore_tar_archive($archive, $vmid, $user, $opts);
6617     } else {
6618         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6619     }
6620 }
6621
6622 # hepler to remove disks that will not be used after restore
6623 my $restore_cleanup_oldconf = sub {
6624     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6625
6626     my $kept_disks = {};
6627
6628     PVE::QemuConfig->foreach_volume($oldconf, sub {
6629         my ($ds, $drive) = @_;
6630
6631         return if drive_is_cdrom($drive, 1);
6632
6633         my $volid = $drive->{file};
6634         return if !$volid || $volid =~ m|^/|;
6635
6636         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6637         return if !$path || !$owner || ($owner != $vmid);
6638
6639         # Note: only delete disk we want to restore
6640         # other volumes will become unused
6641         if ($virtdev_hash->{$ds}) {
6642             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6643             if (my $err = $@) {
6644                 warn $err;
6645             }
6646         } else {
6647             $kept_disks->{$volid} = 1;
6648         }
6649     });
6650
6651     # after the restore we have no snapshots anymore
6652     for my $snapname (keys $oldconf->{snapshots}->%*) {
6653         my $snap = $oldconf->{snapshots}->{$snapname};
6654         if ($snap->{vmstate}) {
6655             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6656             if (my $err = $@) {
6657                 warn $err;
6658             }
6659         }
6660
6661         for my $volid (keys $kept_disks->%*) {
6662             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6663             warn $@ if $@;
6664         }
6665     }
6666 };
6667
6668 # Helper to parse vzdump backup device hints
6669 #
6670 # $rpcenv: Environment, used to ckeck storage permissions
6671 # $user: User ID, to check storage permissions
6672 # $storecfg: Storage configuration
6673 # $fh: the file handle for reading the configuration
6674 # $devinfo: should contain device sizes for all backu-up'ed devices
6675 # $options: backup options (pool, default storage)
6676 #
6677 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6678 my $parse_backup_hints = sub {
6679     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6680
6681     my $check_storage = sub { # assert if an image can be allocate
6682         my ($storeid, $scfg) = @_;
6683         die "Content type 'images' is not available on storage '$storeid'\n"
6684             if !$scfg->{content}->{images};
6685         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6686             if $user ne 'root@pam';
6687     };
6688
6689     my $virtdev_hash = {};
6690     while (defined(my $line = <$fh>)) {
6691         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6692             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6693             die "archive does not contain data for drive '$virtdev'\n"
6694                 if !$devinfo->{$devname};
6695
6696             if (defined($options->{storage})) {
6697                 $storeid = $options->{storage} || 'local';
6698             } elsif (!$storeid) {
6699                 $storeid = 'local';
6700             }
6701             $format = 'raw' if !$format;
6702             $devinfo->{$devname}->{devname} = $devname;
6703             $devinfo->{$devname}->{virtdev} = $virtdev;
6704             $devinfo->{$devname}->{format} = $format;
6705             $devinfo->{$devname}->{storeid} = $storeid;
6706
6707             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6708             $check_storage->($storeid, $scfg); # permission and content type check
6709
6710             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6711         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6712             my $virtdev = $1;
6713             my $drive = parse_drive($virtdev, $2);
6714
6715             if (drive_is_cloudinit($drive)) {
6716                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6717                 $storeid = $options->{storage} if defined ($options->{storage});
6718                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6719                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6720
6721                 $check_storage->($storeid, $scfg); # permission and content type check
6722
6723                 $virtdev_hash->{$virtdev} = {
6724                     format => $format,
6725                     storeid => $storeid,
6726                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6727                     is_cloudinit => 1,
6728                 };
6729             }
6730         }
6731     }
6732
6733     return $virtdev_hash;
6734 };
6735
6736 # Helper to allocate and activate all volumes required for a restore
6737 #
6738 # $storecfg: Storage configuration
6739 # $virtdev_hash: as returned by parse_backup_hints()
6740 #
6741 # Returns: { $virtdev => $volid }
6742 my $restore_allocate_devices = sub {
6743     my ($storecfg, $virtdev_hash, $vmid) = @_;
6744
6745     my $map = {};
6746     foreach my $virtdev (sort keys %$virtdev_hash) {
6747         my $d = $virtdev_hash->{$virtdev};
6748         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6749         my $storeid = $d->{storeid};
6750         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6751
6752         # test if requested format is supported
6753         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6754         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6755         $d->{format} = $defFormat if !$supported;
6756
6757         my $name;
6758         if ($d->{is_cloudinit}) {
6759             $name = "vm-$vmid-cloudinit";
6760             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6761             if ($scfg->{path}) {
6762                 $name .= ".$d->{format}";
6763             }
6764         }
6765
6766         my $volid = PVE::Storage::vdisk_alloc(
6767             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6768
6769         print STDERR "new volume ID is '$volid'\n";
6770         $d->{volid} = $volid;
6771
6772         PVE::Storage::activate_volumes($storecfg, [$volid]);
6773
6774         $map->{$virtdev} = $volid;
6775     }
6776
6777     return $map;
6778 };
6779
6780 sub restore_update_config_line {
6781     my ($cookie, $map, $line, $unique) = @_;
6782
6783     return '' if $line =~ m/^\#qmdump\#/;
6784     return '' if $line =~ m/^\#vzdump\#/;
6785     return '' if $line =~ m/^lock:/;
6786     return '' if $line =~ m/^unused\d+:/;
6787     return '' if $line =~ m/^parent:/;
6788
6789     my $res = '';
6790
6791     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6792     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6793         # try to convert old 1.X settings
6794         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6795         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6796             my ($model, $macaddr) = split(/\=/, $devconfig);
6797             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6798             my $net = {
6799                 model => $model,
6800                 bridge => "vmbr$ind",
6801                 macaddr => $macaddr,
6802             };
6803             my $netstr = print_net($net);
6804
6805             $res .= "net$cookie->{netcount}: $netstr\n";
6806             $cookie->{netcount}++;
6807         }
6808     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6809         my ($id, $netstr) = ($1, $2);
6810         my $net = parse_net($netstr);
6811         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6812         $netstr = print_net($net);
6813         $res .= "$id: $netstr\n";
6814     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6815         my $virtdev = $1;
6816         my $value = $3;
6817         my $di = parse_drive($virtdev, $value);
6818         if (defined($di->{backup}) && !$di->{backup}) {
6819             $res .= "#$line";
6820         } elsif ($map->{$virtdev}) {
6821             delete $di->{format}; # format can change on restore
6822             $di->{file} = $map->{$virtdev};
6823             $value = print_drive($di);
6824             $res .= "$virtdev: $value\n";
6825         } else {
6826             $res .= $line;
6827         }
6828     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6829         my $vmgenid = $1;
6830         if ($vmgenid ne '0') {
6831             # always generate a new vmgenid if there was a valid one setup
6832             $vmgenid = generate_uuid();
6833         }
6834         $res .= "vmgenid: $vmgenid\n";
6835     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6836         my ($uuid, $uuid_str);
6837         UUID::generate($uuid);
6838         UUID::unparse($uuid, $uuid_str);
6839         my $smbios1 = parse_smbios1($2);
6840         $smbios1->{uuid} = $uuid_str;
6841         $res .= $1.print_smbios1($smbios1)."\n";
6842     } else {
6843         $res .= $line;
6844     }
6845
6846     return $res;
6847 }
6848
6849 my $restore_deactivate_volumes = sub {
6850     my ($storecfg, $virtdev_hash) = @_;
6851
6852     my $vollist = [];
6853     for my $dev (values $virtdev_hash->%*) {
6854         push $vollist->@*, $dev->{volid} if $dev->{volid};
6855     }
6856
6857     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6858     print STDERR $@ if $@;
6859 };
6860
6861 my $restore_destroy_volumes = sub {
6862     my ($storecfg, $virtdev_hash) = @_;
6863
6864     for my $dev (values $virtdev_hash->%*) {
6865         my $volid = $dev->{volid} or next;
6866         eval {
6867             PVE::Storage::vdisk_free($storecfg, $volid);
6868             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6869         };
6870         print STDERR "unable to cleanup '$volid' - $@" if $@;
6871     }
6872 };
6873
6874 sub restore_merge_config {
6875     my ($filename, $backup_conf_raw, $override_conf) = @_;
6876
6877     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6878     for my $key (keys $override_conf->%*) {
6879         $backup_conf->{$key} = $override_conf->{$key};
6880     }
6881
6882     return $backup_conf;
6883 }
6884
6885 sub scan_volids {
6886     my ($cfg, $vmid) = @_;
6887
6888     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6889
6890     my $volid_hash = {};
6891     foreach my $storeid (keys %$info) {
6892         foreach my $item (@{$info->{$storeid}}) {
6893             next if !($item->{volid} && $item->{size});
6894             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6895             $volid_hash->{$item->{volid}} = $item;
6896         }
6897     }
6898
6899     return $volid_hash;
6900 }
6901
6902 sub update_disk_config {
6903     my ($vmid, $conf, $volid_hash) = @_;
6904
6905     my $changes;
6906     my $prefix = "VM $vmid";
6907
6908     # used and unused disks
6909     my $referenced = {};
6910
6911     # Note: it is allowed to define multiple storages with same path (alias), so
6912     # we need to check both 'volid' and real 'path' (two different volid can point
6913     # to the same path).
6914
6915     my $referencedpath = {};
6916
6917     # update size info
6918     PVE::QemuConfig->foreach_volume($conf, sub {
6919         my ($opt, $drive) = @_;
6920
6921         my $volid = $drive->{file};
6922         return if !$volid;
6923         my $volume = $volid_hash->{$volid};
6924
6925         # mark volid as "in-use" for next step
6926         $referenced->{$volid} = 1;
6927         if ($volume && (my $path = $volume->{path})) {
6928             $referencedpath->{$path} = 1;
6929         }
6930
6931         return if drive_is_cdrom($drive);
6932         return if !$volume;
6933
6934         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6935         if (defined($updated)) {
6936             $changes = 1;
6937             $conf->{$opt} = print_drive($updated);
6938             print "$prefix ($opt): $msg\n";
6939         }
6940     });
6941
6942     # remove 'unusedX' entry if volume is used
6943     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6944         my ($opt, $drive) = @_;
6945
6946         my $volid = $drive->{file};
6947         return if !$volid;
6948
6949         my $path;
6950         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6951         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6952             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6953             $changes = 1;
6954             delete $conf->{$opt};
6955         }
6956
6957         $referenced->{$volid} = 1;
6958         $referencedpath->{$path} = 1 if $path;
6959     });
6960
6961     foreach my $volid (sort keys %$volid_hash) {
6962         next if $volid =~ m/vm-$vmid-state-/;
6963         next if $referenced->{$volid};
6964         my $path = $volid_hash->{$volid}->{path};
6965         next if !$path; # just to be sure
6966         next if $referencedpath->{$path};
6967         $changes = 1;
6968         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6969         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6970         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6971     }
6972
6973     return $changes;
6974 }
6975
6976 sub rescan {
6977     my ($vmid, $nolock, $dryrun) = @_;
6978
6979     my $cfg = PVE::Storage::config();
6980
6981     print "rescan volumes...\n";
6982     my $volid_hash = scan_volids($cfg, $vmid);
6983
6984     my $updatefn =  sub {
6985         my ($vmid) = @_;
6986
6987         my $conf = PVE::QemuConfig->load_config($vmid);
6988
6989         PVE::QemuConfig->check_lock($conf);
6990
6991         my $vm_volids = {};
6992         foreach my $volid (keys %$volid_hash) {
6993             my $info = $volid_hash->{$volid};
6994             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6995         }
6996
6997         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6998
6999         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
7000     };
7001
7002     if (defined($vmid)) {
7003         if ($nolock) {
7004             &$updatefn($vmid);
7005         } else {
7006             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
7007         }
7008     } else {
7009         my $vmlist = config_list();
7010         foreach my $vmid (keys %$vmlist) {
7011             if ($nolock) {
7012                 &$updatefn($vmid);
7013             } else {
7014                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
7015             }
7016         }
7017     }
7018 }
7019
7020 sub restore_proxmox_backup_archive {
7021     my ($archive, $vmid, $user, $options) = @_;
7022
7023     my $storecfg = PVE::Storage::config();
7024
7025     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
7026     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7027
7028     my $fingerprint = $scfg->{fingerprint};
7029     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
7030
7031     my $repo = PVE::PBSClient::get_repository($scfg);
7032     my $namespace = $scfg->{namespace};
7033
7034     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7035     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7036     local $ENV{PBS_PASSWORD} = $password;
7037     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7038
7039     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7040         PVE::Storage::parse_volname($storecfg, $archive);
7041
7042     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7043
7044     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7045
7046     my $tmpdir = "/var/tmp/vzdumptmp$$";
7047     rmtree $tmpdir;
7048     mkpath $tmpdir;
7049
7050     my $conffile = PVE::QemuConfig->config_file($vmid);
7051      # disable interrupts (always do cleanups)
7052     local $SIG{INT} =
7053         local $SIG{TERM} =
7054         local $SIG{QUIT} =
7055         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7056
7057     # Note: $oldconf is undef if VM does not exists
7058     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7059     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7060     my $new_conf_raw = '';
7061
7062     my $rpcenv = PVE::RPCEnvironment::get();
7063     my $devinfo = {}; # info about drives included in backup
7064     my $virtdev_hash = {}; # info about allocated drives
7065
7066     eval {
7067         # enable interrupts
7068         local $SIG{INT} =
7069             local $SIG{TERM} =
7070             local $SIG{QUIT} =
7071             local $SIG{HUP} =
7072             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7073
7074         my $cfgfn = "$tmpdir/qemu-server.conf";
7075         my $firewall_config_fn = "$tmpdir/fw.conf";
7076         my $index_fn = "$tmpdir/index.json";
7077
7078         my $cmd = "restore";
7079
7080         my $param = [$pbs_backup_name, "index.json", $index_fn];
7081         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7082         my $index = PVE::Tools::file_get_contents($index_fn);
7083         $index = decode_json($index);
7084
7085         foreach my $info (@{$index->{files}}) {
7086             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7087                 my $devname = $1;
7088                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7089                     $devinfo->{$devname}->{size} = $1;
7090                 } else {
7091                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7092                 }
7093             }
7094         }
7095
7096         my $is_qemu_server_backup = scalar(
7097             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7098         );
7099         if (!$is_qemu_server_backup) {
7100             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7101         }
7102         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7103
7104         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7105         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7106
7107         if ($has_firewall_config) {
7108             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7109             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7110
7111             my $pve_firewall_dir = '/etc/pve/firewall';
7112             mkdir $pve_firewall_dir; # make sure the dir exists
7113             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7114         }
7115
7116         my $fh = IO::File->new($cfgfn, "r") ||
7117             die "unable to read qemu-server.conf - $!\n";
7118
7119         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7120
7121         # fixme: rate limit?
7122
7123         # create empty/temp config
7124         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7125
7126         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7127
7128         # allocate volumes
7129         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7130
7131         foreach my $virtdev (sort keys %$virtdev_hash) {
7132             my $d = $virtdev_hash->{$virtdev};
7133             next if $d->{is_cloudinit}; # no need to restore cloudinit
7134
7135             # this fails if storage is unavailable
7136             my $volid = $d->{volid};
7137             my $path = PVE::Storage::path($storecfg, $volid);
7138
7139             # for live-restore we only want to preload the efidisk and TPM state
7140             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7141
7142             my @ns_arg;
7143             if (defined(my $ns = $scfg->{namespace})) {
7144                 @ns_arg = ('--ns', $ns);
7145             }
7146
7147             my $pbs_restore_cmd = [
7148                 '/usr/bin/pbs-restore',
7149                 '--repository', $repo,
7150                 @ns_arg,
7151                 $pbs_backup_name,
7152                 "$d->{devname}.img.fidx",
7153                 $path,
7154                 '--verbose',
7155                 ];
7156
7157             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7158             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7159
7160             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7161                 push @$pbs_restore_cmd, '--skip-zero';
7162             }
7163
7164             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7165             print "restore proxmox backup image: $dbg_cmdstring\n";
7166             run_command($pbs_restore_cmd);
7167         }
7168
7169         $fh->seek(0, 0) || die "seek failed - $!\n";
7170
7171         my $cookie = { netcount => 0 };
7172         while (defined(my $line = <$fh>)) {
7173             $new_conf_raw .= restore_update_config_line(
7174                 $cookie,
7175                 $map,
7176                 $line,
7177                 $options->{unique},
7178             );
7179         }
7180
7181         $fh->close();
7182     };
7183     my $err = $@;
7184
7185     if ($err || !$options->{live}) {
7186         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7187     }
7188
7189     rmtree $tmpdir;
7190
7191     if ($err) {
7192         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7193         die $err;
7194     }
7195
7196     if ($options->{live}) {
7197         # keep lock during live-restore
7198         $new_conf_raw .= "\nlock: create";
7199     }
7200
7201     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7202     check_restore_permissions($rpcenv, $user, $new_conf);
7203     PVE::QemuConfig->write_config($vmid, $new_conf);
7204
7205     eval { rescan($vmid, 1); };
7206     warn $@ if $@;
7207
7208     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7209
7210     if ($options->{live}) {
7211         # enable interrupts
7212         local $SIG{INT} =
7213             local $SIG{TERM} =
7214             local $SIG{QUIT} =
7215             local $SIG{HUP} =
7216             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7217
7218         my $conf = PVE::QemuConfig->load_config($vmid);
7219         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7220
7221         # these special drives are already restored before start
7222         delete $devinfo->{'drive-efidisk0'};
7223         delete $devinfo->{'drive-tpmstate0-backup'};
7224
7225         my $pbs_opts = {
7226             repo => $repo,
7227             keyfile => $keyfile,
7228             snapshot => $pbs_backup_name,
7229             namespace => $namespace,
7230         };
7231         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7232
7233         PVE::QemuConfig->remove_lock($vmid, "create");
7234     }
7235 }
7236
7237 sub pbs_live_restore {
7238     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7239
7240     print "starting VM for live-restore\n";
7241     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7242
7243     my $pbs_backing = {};
7244     for my $ds (keys %$restored_disks) {
7245         $ds =~ m/^drive-(.*)$/;
7246         my $confname = $1;
7247         $pbs_backing->{$confname} = {
7248             repository => $opts->{repo},
7249             snapshot => $opts->{snapshot},
7250             archive => "$ds.img.fidx",
7251         };
7252         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7253         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7254
7255         my $drive = parse_drive($confname, $conf->{$confname});
7256         print "restoring '$ds' to '$drive->{file}'\n";
7257     }
7258
7259     my $drives_streamed = 0;
7260     eval {
7261         # make sure HA doesn't interrupt our restore by stopping the VM
7262         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7263             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7264         }
7265
7266         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7267         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7268         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7269
7270         my $qmeventd_fd = register_qmeventd_handle($vmid);
7271
7272         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7273         # this will effectively collapse the backing image chain consisting of
7274         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7275         # removes itself once all backing images vanish with 'auto-remove=on')
7276         my $jobs = {};
7277         for my $ds (sort keys %$restored_disks) {
7278             my $job_id = "restore-$ds";
7279             mon_cmd($vmid, 'block-stream',
7280                 'job-id' => $job_id,
7281                 device => "$ds",
7282             );
7283             $jobs->{$job_id} = {};
7284         }
7285
7286         mon_cmd($vmid, 'cont');
7287         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7288
7289         print "restore-drive jobs finished successfully, removing all tracking block devices"
7290             ." to disconnect from Proxmox Backup Server\n";
7291
7292         for my $ds (sort keys %$restored_disks) {
7293             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7294         }
7295
7296         close($qmeventd_fd);
7297     };
7298
7299     my $err = $@;
7300
7301     if ($err) {
7302         warn "An error occurred during live-restore: $err\n";
7303         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7304         die "live-restore failed\n";
7305     }
7306 }
7307
7308 sub restore_vma_archive {
7309     my ($archive, $vmid, $user, $opts, $comp) = @_;
7310
7311     my $readfrom = $archive;
7312
7313     my $cfg = PVE::Storage::config();
7314     my $commands = [];
7315     my $bwlimit = $opts->{bwlimit};
7316
7317     my $dbg_cmdstring = '';
7318     my $add_pipe = sub {
7319         my ($cmd) = @_;
7320         push @$commands, $cmd;
7321         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7322         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7323         $readfrom = '-';
7324     };
7325
7326     my $input = undef;
7327     if ($archive eq '-') {
7328         $input = '<&STDIN';
7329     } else {
7330         # If we use a backup from a PVE defined storage we also consider that
7331         # storage's rate limit:
7332         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7333         if (defined($volid)) {
7334             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7335             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7336             if ($readlimit) {
7337                 print STDERR "applying read rate limit: $readlimit\n";
7338                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7339                 $add_pipe->($cstream);
7340             }
7341         }
7342     }
7343
7344     if ($comp) {
7345         my $info = PVE::Storage::decompressor_info('vma', $comp);
7346         my $cmd = $info->{decompressor};
7347         push @$cmd, $readfrom;
7348         $add_pipe->($cmd);
7349     }
7350
7351     my $tmpdir = "/var/tmp/vzdumptmp$$";
7352     rmtree $tmpdir;
7353
7354     # disable interrupts (always do cleanups)
7355     local $SIG{INT} =
7356         local $SIG{TERM} =
7357         local $SIG{QUIT} =
7358         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7359
7360     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7361     POSIX::mkfifo($mapfifo, 0600);
7362     my $fifofh;
7363     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7364
7365     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7366
7367     my $oldtimeout;
7368     my $timeout = 5;
7369
7370     my $devinfo = {}; # info about drives included in backup
7371     my $virtdev_hash = {}; # info about allocated drives
7372
7373     my $rpcenv = PVE::RPCEnvironment::get();
7374
7375     my $conffile = PVE::QemuConfig->config_file($vmid);
7376
7377     # Note: $oldconf is undef if VM does not exist
7378     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7379     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7380     my $new_conf_raw = '';
7381
7382     my %storage_limits;
7383
7384     my $print_devmap = sub {
7385         my $cfgfn = "$tmpdir/qemu-server.conf";
7386
7387         # we can read the config - that is already extracted
7388         my $fh = IO::File->new($cfgfn, "r") ||
7389             die "unable to read qemu-server.conf - $!\n";
7390
7391         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7392         if (-f $fwcfgfn) {
7393             my $pve_firewall_dir = '/etc/pve/firewall';
7394             mkdir $pve_firewall_dir; # make sure the dir exists
7395             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7396         }
7397
7398         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7399
7400         foreach my $info (values %{$virtdev_hash}) {
7401             my $storeid = $info->{storeid};
7402             next if defined($storage_limits{$storeid});
7403
7404             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7405             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7406             $storage_limits{$storeid} = $limit * 1024;
7407         }
7408
7409         foreach my $devname (keys %$devinfo) {
7410             die "found no device mapping information for device '$devname'\n"
7411                 if !$devinfo->{$devname}->{virtdev};
7412         }
7413
7414         # create empty/temp config
7415         if ($oldconf) {
7416             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7417             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7418         }
7419
7420         # allocate volumes
7421         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7422
7423         # print restore information to $fifofh
7424         foreach my $virtdev (sort keys %$virtdev_hash) {
7425             my $d = $virtdev_hash->{$virtdev};
7426             next if $d->{is_cloudinit}; # no need to restore cloudinit
7427
7428             my $storeid = $d->{storeid};
7429             my $volid = $d->{volid};
7430
7431             my $map_opts = '';
7432             if (my $limit = $storage_limits{$storeid}) {
7433                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7434             }
7435
7436             my $write_zeros = 1;
7437             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7438                 $write_zeros = 0;
7439             }
7440
7441             my $path = PVE::Storage::path($cfg, $volid);
7442
7443             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7444
7445             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7446         }
7447
7448         $fh->seek(0, 0) || die "seek failed - $!\n";
7449
7450         my $cookie = { netcount => 0 };
7451         while (defined(my $line = <$fh>)) {
7452             $new_conf_raw .= restore_update_config_line(
7453                 $cookie,
7454                 $map,
7455                 $line,
7456                 $opts->{unique},
7457             );
7458         }
7459
7460         $fh->close();
7461     };
7462
7463     eval {
7464         # enable interrupts
7465         local $SIG{INT} =
7466             local $SIG{TERM} =
7467             local $SIG{QUIT} =
7468             local $SIG{HUP} =
7469             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7470         local $SIG{ALRM} = sub { die "got timeout\n"; };
7471
7472         $oldtimeout = alarm($timeout);
7473
7474         my $parser = sub {
7475             my $line = shift;
7476
7477             print "$line\n";
7478
7479             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7480                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7481                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7482             } elsif ($line =~ m/^CTIME: /) {
7483                 # we correctly received the vma config, so we can disable
7484                 # the timeout now for disk allocation (set to 10 minutes, so
7485                 # that we always timeout if something goes wrong)
7486                 alarm(600);
7487                 &$print_devmap();
7488                 print $fifofh "done\n";
7489                 my $tmp = $oldtimeout || 0;
7490                 $oldtimeout = undef;
7491                 alarm($tmp);
7492                 close($fifofh);
7493                 $fifofh = undef;
7494             }
7495         };
7496
7497         print "restore vma archive: $dbg_cmdstring\n";
7498         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7499     };
7500     my $err = $@;
7501
7502     alarm($oldtimeout) if $oldtimeout;
7503
7504     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7505
7506     close($fifofh) if $fifofh;
7507     unlink $mapfifo;
7508     rmtree $tmpdir;
7509
7510     if ($err) {
7511         $restore_destroy_volumes->($cfg, $virtdev_hash);
7512         die $err;
7513     }
7514
7515     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7516     check_restore_permissions($rpcenv, $user, $new_conf);
7517     PVE::QemuConfig->write_config($vmid, $new_conf);
7518
7519     eval { rescan($vmid, 1); };
7520     warn $@ if $@;
7521
7522     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7523 }
7524
7525 sub restore_tar_archive {
7526     my ($archive, $vmid, $user, $opts) = @_;
7527
7528     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7529         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7530         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7531     }
7532
7533     if ($archive ne '-') {
7534         my $firstfile = tar_archive_read_firstfile($archive);
7535         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7536             if $firstfile ne 'qemu-server.conf';
7537     }
7538
7539     my $storecfg = PVE::Storage::config();
7540
7541     # avoid zombie disks when restoring over an existing VM -> cleanup first
7542     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7543     # skiplock=1 because qmrestore has set the 'create' lock itself already
7544     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7545     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7546
7547     my $tocmd = "/usr/lib/qemu-server/qmextract";
7548
7549     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7550     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7551     $tocmd .= ' --prealloc' if $opts->{prealloc};
7552     $tocmd .= ' --info' if $opts->{info};
7553
7554     # tar option "xf" does not autodetect compression when read from STDIN,
7555     # so we pipe to zcat
7556     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7557         PVE::Tools::shellquote("--to-command=$tocmd");
7558
7559     my $tmpdir = "/var/tmp/vzdumptmp$$";
7560     mkpath $tmpdir;
7561
7562     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7563     local $ENV{VZDUMP_VMID} = $vmid;
7564     local $ENV{VZDUMP_USER} = $user;
7565
7566     my $conffile = PVE::QemuConfig->config_file($vmid);
7567     my $new_conf_raw = '';
7568
7569     # disable interrupts (always do cleanups)
7570     local $SIG{INT} =
7571         local $SIG{TERM} =
7572         local $SIG{QUIT} =
7573         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7574
7575     eval {
7576         # enable interrupts
7577         local $SIG{INT} =
7578             local $SIG{TERM} =
7579             local $SIG{QUIT} =
7580             local $SIG{HUP} =
7581             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7582
7583         if ($archive eq '-') {
7584             print "extracting archive from STDIN\n";
7585             run_command($cmd, input => "<&STDIN");
7586         } else {
7587             print "extracting archive '$archive'\n";
7588             run_command($cmd);
7589         }
7590
7591         return if $opts->{info};
7592
7593         # read new mapping
7594         my $map = {};
7595         my $statfile = "$tmpdir/qmrestore.stat";
7596         if (my $fd = IO::File->new($statfile, "r")) {
7597             while (defined (my $line = <$fd>)) {
7598                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7599                     $map->{$1} = $2 if $1;
7600                 } else {
7601                     print STDERR "unable to parse line in statfile - $line\n";
7602                 }
7603             }
7604             $fd->close();
7605         }
7606
7607         my $confsrc = "$tmpdir/qemu-server.conf";
7608
7609         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7610
7611         my $cookie = { netcount => 0 };
7612         while (defined (my $line = <$srcfd>)) {
7613             $new_conf_raw .= restore_update_config_line(
7614                 $cookie,
7615                 $map,
7616                 $line,
7617                 $opts->{unique},
7618             );
7619         }
7620
7621         $srcfd->close();
7622     };
7623     if (my $err = $@) {
7624         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7625         die $err;
7626     }
7627
7628     rmtree $tmpdir;
7629
7630     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7631
7632     PVE::Cluster::cfs_update(); # make sure we read new file
7633
7634     eval { rescan($vmid, 1); };
7635     warn $@ if $@;
7636 };
7637
7638 sub foreach_storage_used_by_vm {
7639     my ($conf, $func) = @_;
7640
7641     my $sidhash = {};
7642
7643     PVE::QemuConfig->foreach_volume($conf, sub {
7644         my ($ds, $drive) = @_;
7645         return if drive_is_cdrom($drive);
7646
7647         my $volid = $drive->{file};
7648
7649         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7650         $sidhash->{$sid} = $sid if $sid;
7651     });
7652
7653     foreach my $sid (sort keys %$sidhash) {
7654         &$func($sid);
7655     }
7656 }
7657
7658 my $qemu_snap_storage = {
7659     rbd => 1,
7660 };
7661 sub do_snapshots_with_qemu {
7662     my ($storecfg, $volid, $deviceid) = @_;
7663
7664     return if $deviceid =~ m/tpmstate0/;
7665
7666     my $storage_name = PVE::Storage::parse_volume_id($volid);
7667     my $scfg = $storecfg->{ids}->{$storage_name};
7668     die "could not find storage '$storage_name'\n" if !defined($scfg);
7669
7670     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7671         return 1;
7672     }
7673
7674     if ($volid =~ m/\.(qcow2|qed)$/){
7675         return 1;
7676     }
7677
7678     return;
7679 }
7680
7681 sub qga_check_running {
7682     my ($vmid, $nowarn) = @_;
7683
7684     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7685     if ($@) {
7686         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7687         return 0;
7688     }
7689     return 1;
7690 }
7691
7692 sub template_create {
7693     my ($vmid, $conf, $disk) = @_;
7694
7695     my $storecfg = PVE::Storage::config();
7696
7697     PVE::QemuConfig->foreach_volume($conf, sub {
7698         my ($ds, $drive) = @_;
7699
7700         return if drive_is_cdrom($drive);
7701         return if $disk && $ds ne $disk;
7702
7703         my $volid = $drive->{file};
7704         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7705
7706         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7707         $drive->{file} = $voliddst;
7708         $conf->{$ds} = print_drive($drive);
7709         PVE::QemuConfig->write_config($vmid, $conf);
7710     });
7711 }
7712
7713 sub convert_iscsi_path {
7714     my ($path) = @_;
7715
7716     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7717         my $portal = $1;
7718         my $target = $2;
7719         my $lun = $3;
7720
7721         my $initiator_name = get_initiator_name();
7722
7723         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7724                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7725     }
7726
7727     die "cannot convert iscsi path '$path', unkown format\n";
7728 }
7729
7730 sub qemu_img_convert {
7731     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7732
7733     my $storecfg = PVE::Storage::config();
7734     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7735     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7736
7737     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7738
7739     my $cachemode;
7740     my $src_path;
7741     my $src_is_iscsi = 0;
7742     my $src_format;
7743
7744     if ($src_storeid) {
7745         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7746         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7747         $src_format = qemu_img_format($src_scfg, $src_volname);
7748         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7749         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7750         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7751     } elsif (-f $src_volid || -b $src_volid) {
7752         $src_path = $src_volid;
7753         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7754             $src_format = $1;
7755         }
7756     }
7757
7758     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7759
7760     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7761     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7762     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7763     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7764
7765     my $cmd = [];
7766     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7767     push @$cmd, '-l', "snapshot.name=$snapname"
7768         if $snapname && $src_format && $src_format eq "qcow2";
7769     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7770     push @$cmd, '-T', $cachemode if defined($cachemode);
7771     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7772
7773     if ($src_is_iscsi) {
7774         push @$cmd, '--image-opts';
7775         $src_path = convert_iscsi_path($src_path);
7776     } elsif ($src_format) {
7777         push @$cmd, '-f', $src_format;
7778     }
7779
7780     if ($dst_is_iscsi) {
7781         push @$cmd, '--target-image-opts';
7782         $dst_path = convert_iscsi_path($dst_path);
7783     } else {
7784         push @$cmd, '-O', $dst_format;
7785     }
7786
7787     push @$cmd, $src_path;
7788
7789     if (!$dst_is_iscsi && $is_zero_initialized) {
7790         push @$cmd, "zeroinit:$dst_path";
7791     } else {
7792         push @$cmd, $dst_path;
7793     }
7794
7795     my $parser = sub {
7796         my $line = shift;
7797         if($line =~ m/\((\S+)\/100\%\)/){
7798             my $percent = $1;
7799             my $transferred = int($size * $percent / 100);
7800             my $total_h = render_bytes($size, 1);
7801             my $transferred_h = render_bytes($transferred, 1);
7802
7803             print "transferred $transferred_h of $total_h ($percent%)\n";
7804         }
7805
7806     };
7807
7808     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7809     my $err = $@;
7810     die "copy failed: $err" if $err;
7811 }
7812
7813 sub qemu_img_format {
7814     my ($scfg, $volname) = @_;
7815
7816     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7817         return $1;
7818     } else {
7819         return "raw";
7820     }
7821 }
7822
7823 sub qemu_drive_mirror {
7824     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7825
7826     $jobs = {} if !$jobs;
7827
7828     my $qemu_target;
7829     my $format;
7830     $jobs->{"drive-$drive"} = {};
7831
7832     if ($dst_volid =~ /^nbd:/) {
7833         $qemu_target = $dst_volid;
7834         $format = "nbd";
7835     } else {
7836         my $storecfg = PVE::Storage::config();
7837         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7838
7839         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7840
7841         $format = qemu_img_format($dst_scfg, $dst_volname);
7842
7843         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7844
7845         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7846     }
7847
7848     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7849     $opts->{format} = $format if $format;
7850
7851     if (defined($src_bitmap)) {
7852         $opts->{sync} = 'incremental';
7853         $opts->{bitmap} = $src_bitmap;
7854         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7855     }
7856
7857     if (defined($bwlimit)) {
7858         $opts->{speed} = $bwlimit * 1024;
7859         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7860     } else {
7861         print "drive mirror is starting for drive-$drive\n";
7862     }
7863
7864     # if a job already runs for this device we get an error, catch it for cleanup
7865     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7866     if (my $err = $@) {
7867         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7868         warn "$@\n" if $@;
7869         die "mirroring error: $err\n";
7870     }
7871
7872     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7873 }
7874
7875 # $completion can be either
7876 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7877 # 'cancel': wait until all jobs are ready, block-job-cancel them
7878 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7879 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7880 sub qemu_drive_mirror_monitor {
7881     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7882
7883     $completion //= 'complete';
7884     $op //= "mirror";
7885
7886     eval {
7887         my $err_complete = 0;
7888
7889         my $starttime = time ();
7890         while (1) {
7891             die "block job ('$op') timed out\n" if $err_complete > 300;
7892
7893             my $stats = mon_cmd($vmid, "query-block-jobs");
7894             my $ctime = time();
7895
7896             my $running_jobs = {};
7897             for my $stat (@$stats) {
7898                 next if $stat->{type} ne $op;
7899                 $running_jobs->{$stat->{device}} = $stat;
7900             }
7901
7902             my $readycounter = 0;
7903
7904             for my $job_id (sort keys %$jobs) {
7905                 my $job = $running_jobs->{$job_id};
7906
7907                 my $vanished = !defined($job);
7908                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7909                 if($complete || ($vanished && $completion eq 'auto')) {
7910                     print "$job_id: $op-job finished\n";
7911                     delete $jobs->{$job_id};
7912                     next;
7913                 }
7914
7915                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7916
7917                 my $busy = $job->{busy};
7918                 my $ready = $job->{ready};
7919                 if (my $total = $job->{len}) {
7920                     my $transferred = $job->{offset} || 0;
7921                     my $remaining = $total - $transferred;
7922                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7923
7924                     my $duration = $ctime - $starttime;
7925                     my $total_h = render_bytes($total, 1);
7926                     my $transferred_h = render_bytes($transferred, 1);
7927
7928                     my $status = sprintf(
7929                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7930                         render_duration($duration),
7931                     );
7932
7933                     if ($ready) {
7934                         if ($busy) {
7935                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7936                         } else {
7937                             $status .= ", ready";
7938                         }
7939                     }
7940                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7941                     $jobs->{$job_id}->{ready} = $ready;
7942                 }
7943
7944                 $readycounter++ if $job->{ready};
7945             }
7946
7947             last if scalar(keys %$jobs) == 0;
7948
7949             if ($readycounter == scalar(keys %$jobs)) {
7950                 print "all '$op' jobs are ready\n";
7951
7952                 # do the complete later (or has already been done)
7953                 last if $completion eq 'skip' || $completion eq 'auto';
7954
7955                 if ($vmiddst && $vmiddst != $vmid) {
7956                     my $agent_running = $qga && qga_check_running($vmid);
7957                     if ($agent_running) {
7958                         print "freeze filesystem\n";
7959                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7960                         warn $@ if $@;
7961                     } else {
7962                         print "suspend vm\n";
7963                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7964                         warn $@ if $@;
7965                     }
7966
7967                     # if we clone a disk for a new target vm, we don't switch the disk
7968                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7969
7970                     if ($agent_running) {
7971                         print "unfreeze filesystem\n";
7972                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7973                         warn $@ if $@;
7974                     } else {
7975                         print "resume vm\n";
7976                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7977                         warn $@ if $@;
7978                     }
7979
7980                     last;
7981                 } else {
7982
7983                     for my $job_id (sort keys %$jobs) {
7984                         # try to switch the disk if source and destination are on the same guest
7985                         print "$job_id: Completing block job_id...\n";
7986
7987                         my $op;
7988                         if ($completion eq 'complete') {
7989                             $op = 'block-job-complete';
7990                         } elsif ($completion eq 'cancel') {
7991                             $op = 'block-job-cancel';
7992                         } else {
7993                             die "invalid completion value: $completion\n";
7994                         }
7995                         eval { mon_cmd($vmid, $op, device => $job_id) };
7996                         if ($@ =~ m/cannot be completed/) {
7997                             print "$job_id: block job cannot be completed, trying again.\n";
7998                             $err_complete++;
7999                         }else {
8000                             print "$job_id: Completed successfully.\n";
8001                             $jobs->{$job_id}->{complete} = 1;
8002                         }
8003                     }
8004                 }
8005             }
8006             sleep 1;
8007         }
8008     };
8009     my $err = $@;
8010
8011     if ($err) {
8012         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
8013         die "block job ($op) error: $err";
8014     }
8015 }
8016
8017 sub qemu_blockjobs_cancel {
8018     my ($vmid, $jobs) = @_;
8019
8020     foreach my $job (keys %$jobs) {
8021         print "$job: Cancelling block job\n";
8022         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
8023         $jobs->{$job}->{cancel} = 1;
8024     }
8025
8026     while (1) {
8027         my $stats = mon_cmd($vmid, "query-block-jobs");
8028
8029         my $running_jobs = {};
8030         foreach my $stat (@$stats) {
8031             $running_jobs->{$stat->{device}} = $stat;
8032         }
8033
8034         foreach my $job (keys %$jobs) {
8035
8036             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
8037                 print "$job: Done.\n";
8038                 delete $jobs->{$job};
8039             }
8040         }
8041
8042         last if scalar(keys %$jobs) == 0;
8043
8044         sleep 1;
8045     }
8046 }
8047
8048 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8049 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8050 my sub clone_disk_check_io_uring {
8051     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8052
8053     return if !$use_drive_mirror;
8054
8055     # Don't complain when not changing storage.
8056     # Assume if it works for the source, it'll work for the target too.
8057     return if $src_storeid eq $dst_storeid;
8058
8059     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8060     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8061
8062     my $cache_direct = drive_uses_cache_direct($src_drive);
8063
8064     my $src_uses_io_uring;
8065     if ($src_drive->{aio}) {
8066         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8067     } else {
8068         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8069     }
8070
8071     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8072         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8073 }
8074
8075 sub clone_disk {
8076     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8077
8078     my ($vmid, $running) = $source->@{qw(vmid running)};
8079     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8080
8081     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8082     my ($storage, $format) = $dest->@{qw(storage format)};
8083
8084     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8085
8086     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8087         die "cloning from/to EFI disk requires EFI disk\n"
8088             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8089         die "cloning from/to TPM state requires TPM state\n"
8090             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8091
8092         # This would lead to two device nodes in QEMU pointing to the same backing image!
8093         die "cannot change drive name when cloning disk from/to the same VM\n"
8094             if $use_drive_mirror && $vmid == $newvmid;
8095     }
8096
8097     die "cannot move TPM state while VM is running\n"
8098         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8099
8100     my $newvolid;
8101
8102     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8103     print "$src_drivename " if $src_drivename;
8104     print "($drive->{file})\n";
8105
8106     if (!$full) {
8107         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8108         push @$newvollist, $newvolid;
8109     } else {
8110         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8111         my $storeid = $storage || $src_storeid;
8112
8113         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8114
8115         my $name = undef;
8116         my $size = undef;
8117         if (drive_is_cloudinit($drive)) {
8118             $name = "vm-$newvmid-cloudinit";
8119             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8120             if ($scfg->{path}) {
8121                 $name .= ".$dst_format";
8122             }
8123             $snapname = undef;
8124             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8125         } elsif ($dst_drivename eq 'efidisk0') {
8126             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8127         } elsif ($dst_drivename eq 'tpmstate0') {
8128             $dst_format = 'raw';
8129             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8130         } else {
8131             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8132
8133             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8134         }
8135         $newvolid = PVE::Storage::vdisk_alloc(
8136             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8137         );
8138         push @$newvollist, $newvolid;
8139
8140         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8141
8142         if (drive_is_cloudinit($drive)) {
8143             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8144             # if this is the case, we have to complete any block-jobs still there from
8145             # previous drive-mirrors
8146             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8147                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8148             }
8149             goto no_data_clone;
8150         }
8151
8152         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8153         if ($use_drive_mirror) {
8154             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8155                 $completion, $qga, $bwlimit);
8156         } else {
8157             if ($dst_drivename eq 'efidisk0') {
8158                 # the relevant data on the efidisk may be smaller than the source
8159                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8160                 # that is given by the OVMF_VARS.fd
8161                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8162                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8163
8164                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8165
8166                 # better for Ceph if block size is not too small, see bug #3324
8167                 my $bs = 1024*1024;
8168
8169                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8170
8171                 if ($src_format eq 'qcow2' && $snapname) {
8172                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8173                         if !min_version(kvm_user_version(), 6, 2);
8174                     push $cmd->@*, '-l', $snapname;
8175                 }
8176                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8177                 run_command($cmd);
8178             } else {
8179                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8180             }
8181         }
8182     }
8183
8184 no_data_clone:
8185     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8186
8187     my $disk = dclone($drive);
8188     delete $disk->{format};
8189     $disk->{file} = $newvolid;
8190     $disk->{size} = $size if defined($size);
8191
8192     return $disk;
8193 }
8194
8195 sub get_running_qemu_version {
8196     my ($vmid) = @_;
8197     my $res = mon_cmd($vmid, "query-version");
8198     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8199 }
8200
8201 sub qemu_use_old_bios_files {
8202     my ($machine_type) = @_;
8203
8204     return if !$machine_type;
8205
8206     my $use_old_bios_files = undef;
8207
8208     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8209         $machine_type = $1;
8210         $use_old_bios_files = 1;
8211     } else {
8212         my $version = extract_version($machine_type, kvm_user_version());
8213         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8214         # load new efi bios files on migration. So this hack is required to allow
8215         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8216         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8217         $use_old_bios_files = !min_version($version, 2, 4);
8218     }
8219
8220     return ($use_old_bios_files, $machine_type);
8221 }
8222
8223 sub get_efivars_size {
8224     my ($conf, $efidisk) = @_;
8225
8226     my $arch = get_vm_arch($conf);
8227     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8228     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8229     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8230     return -s $ovmf_vars;
8231 }
8232
8233 sub update_efidisk_size {
8234     my ($conf) = @_;
8235
8236     return if !defined($conf->{efidisk0});
8237
8238     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8239     $disk->{size} = get_efivars_size($conf);
8240     $conf->{efidisk0} = print_drive($disk);
8241
8242     return;
8243 }
8244
8245 sub update_tpmstate_size {
8246     my ($conf) = @_;
8247
8248     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8249     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8250     $conf->{tpmstate0} = print_drive($disk);
8251 }
8252
8253 sub create_efidisk($$$$$$$) {
8254     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8255
8256     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8257
8258     my $vars_size_b = -s $ovmf_vars;
8259     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8260     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8261     PVE::Storage::activate_volumes($storecfg, [$volid]);
8262
8263     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8264     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8265
8266     return ($volid, $size/1024);
8267 }
8268
8269 sub vm_iothreads_list {
8270     my ($vmid) = @_;
8271
8272     my $res = mon_cmd($vmid, 'query-iothreads');
8273
8274     my $iothreads = {};
8275     foreach my $iothread (@$res) {
8276         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8277     }
8278
8279     return $iothreads;
8280 }
8281
8282 sub scsihw_infos {
8283     my ($conf, $drive) = @_;
8284
8285     my $maxdev = 0;
8286
8287     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8288         $maxdev = 7;
8289     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8290         $maxdev = 1;
8291     } else {
8292         $maxdev = 256;
8293     }
8294
8295     my $controller = int($drive->{index} / $maxdev);
8296     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8297         ? "virtioscsi"
8298         : "scsihw";
8299
8300     return ($maxdev, $controller, $controller_prefix);
8301 }
8302
8303 sub resolve_dst_disk_format {
8304         my ($storecfg, $storeid, $src_volname, $format) = @_;
8305         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8306
8307         if (!$format) {
8308             # if no target format is specified, use the source disk format as hint
8309             if ($src_volname) {
8310                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8311                 $format = qemu_img_format($scfg, $src_volname);
8312             } else {
8313                 return $defFormat;
8314             }
8315         }
8316
8317         # test if requested format is supported - else use default
8318         my $supported = grep { $_ eq $format } @$validFormats;
8319         $format = $defFormat if !$supported;
8320         return $format;
8321 }
8322
8323 # NOTE: if this logic changes, please update docs & possibly gui logic
8324 sub find_vmstate_storage {
8325     my ($conf, $storecfg) = @_;
8326
8327     # first, return storage from conf if set
8328     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8329
8330     my ($target, $shared, $local);
8331
8332     foreach_storage_used_by_vm($conf, sub {
8333         my ($sid) = @_;
8334         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8335         my $dst = $scfg->{shared} ? \$shared : \$local;
8336         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8337     });
8338
8339     # second, use shared storage where VM has at least one disk
8340     # third, use local storage where VM has at least one disk
8341     # fall back to local storage
8342     $target = $shared // $local // 'local';
8343
8344     return $target;
8345 }
8346
8347 sub generate_uuid {
8348     my ($uuid, $uuid_str);
8349     UUID::generate($uuid);
8350     UUID::unparse($uuid, $uuid_str);
8351     return $uuid_str;
8352 }
8353
8354 sub generate_smbios1_uuid {
8355     return "uuid=".generate_uuid();
8356 }
8357
8358 sub nbd_stop {
8359     my ($vmid) = @_;
8360
8361     mon_cmd($vmid, 'nbd-server-stop');
8362 }
8363
8364 sub create_reboot_request {
8365     my ($vmid) = @_;
8366     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8367         or die "failed to create reboot trigger file: $!\n";
8368     close($fh);
8369 }
8370
8371 sub clear_reboot_request {
8372     my ($vmid) = @_;
8373     my $path = "/run/qemu-server/$vmid.reboot";
8374     my $res = 0;
8375
8376     $res = unlink($path);
8377     die "could not remove reboot request for $vmid: $!"
8378         if !$res && $! != POSIX::ENOENT;
8379
8380     return $res;
8381 }
8382
8383 sub bootorder_from_legacy {
8384     my ($conf, $bootcfg) = @_;
8385
8386     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8387     my $bootindex_hash = {};
8388     my $i = 1;
8389     foreach my $o (split(//, $boot)) {
8390         $bootindex_hash->{$o} = $i*100;
8391         $i++;
8392     }
8393
8394     my $bootorder = {};
8395
8396     PVE::QemuConfig->foreach_volume($conf, sub {
8397         my ($ds, $drive) = @_;
8398
8399         if (drive_is_cdrom ($drive, 1)) {
8400             if ($bootindex_hash->{d}) {
8401                 $bootorder->{$ds} = $bootindex_hash->{d};
8402                 $bootindex_hash->{d} += 1;
8403             }
8404         } elsif ($bootindex_hash->{c}) {
8405             $bootorder->{$ds} = $bootindex_hash->{c}
8406                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8407             $bootindex_hash->{c} += 1;
8408         }
8409     });
8410
8411     if ($bootindex_hash->{n}) {
8412         for (my $i = 0; $i < $MAX_NETS; $i++) {
8413             my $netname = "net$i";
8414             next if !$conf->{$netname};
8415             $bootorder->{$netname} = $bootindex_hash->{n};
8416             $bootindex_hash->{n} += 1;
8417         }
8418     }
8419
8420     return $bootorder;
8421 }
8422
8423 # Generate default device list for 'boot: order=' property. Matches legacy
8424 # default boot order, but with explicit device names. This is important, since
8425 # the fallback for when neither 'order' nor the old format is specified relies
8426 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8427 sub get_default_bootdevices {
8428     my ($conf) = @_;
8429
8430     my @ret = ();
8431
8432     # harddisk
8433     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8434     push @ret, $first if $first;
8435
8436     # cdrom
8437     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8438     push @ret, $first if $first;
8439
8440     # network
8441     for (my $i = 0; $i < $MAX_NETS; $i++) {
8442         my $netname = "net$i";
8443         next if !$conf->{$netname};
8444         push @ret, $netname;
8445         last;
8446     }
8447
8448     return \@ret;
8449 }
8450
8451 sub device_bootorder {
8452     my ($conf) = @_;
8453
8454     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8455
8456     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8457
8458     my $bootorder = {};
8459     if (!defined($boot) || $boot->{legacy}) {
8460         $bootorder = bootorder_from_legacy($conf, $boot);
8461     } elsif ($boot->{order}) {
8462         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8463         for my $dev (PVE::Tools::split_list($boot->{order})) {
8464             $bootorder->{$dev} = $i++;
8465         }
8466     }
8467
8468     return $bootorder;
8469 }
8470
8471 sub register_qmeventd_handle {
8472     my ($vmid) = @_;
8473
8474     my $fh;
8475     my $peer = "/var/run/qmeventd.sock";
8476     my $count = 0;
8477
8478     for (;;) {
8479         $count++;
8480         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8481         last if $fh;
8482         if ($! != EINTR && $! != EAGAIN) {
8483             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8484         }
8485         if ($count > 4) {
8486             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8487               . "after $count retries\n";
8488         }
8489         usleep(25000);
8490     }
8491
8492     # send handshake to mark VM as backing up
8493     print $fh to_json({vzdump => {vmid => "$vmid"}});
8494
8495     # return handle to be closed later when inhibit is no longer required
8496     return $fh;
8497 }
8498
8499 # bash completion helper
8500
8501 sub complete_backup_archives {
8502     my ($cmdname, $pname, $cvalue) = @_;
8503
8504     my $cfg = PVE::Storage::config();
8505
8506     my $storeid;
8507
8508     if ($cvalue =~ m/^([^:]+):/) {
8509         $storeid = $1;
8510     }
8511
8512     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8513
8514     my $res = [];
8515     foreach my $id (keys %$data) {
8516         foreach my $item (@{$data->{$id}}) {
8517             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8518             push @$res, $item->{volid} if defined($item->{volid});
8519         }
8520     }
8521
8522     return $res;
8523 }
8524
8525 my $complete_vmid_full = sub {
8526     my ($running) = @_;
8527
8528     my $idlist = vmstatus();
8529
8530     my $res = [];
8531
8532     foreach my $id (keys %$idlist) {
8533         my $d = $idlist->{$id};
8534         if (defined($running)) {
8535             next if $d->{template};
8536             next if $running && $d->{status} ne 'running';
8537             next if !$running && $d->{status} eq 'running';
8538         }
8539         push @$res, $id;
8540
8541     }
8542     return $res;
8543 };
8544
8545 sub complete_vmid {
8546     return &$complete_vmid_full();
8547 }
8548
8549 sub complete_vmid_stopped {
8550     return &$complete_vmid_full(0);
8551 }
8552
8553 sub complete_vmid_running {
8554     return &$complete_vmid_full(1);
8555 }
8556
8557 sub complete_storage {
8558
8559     my $cfg = PVE::Storage::config();
8560     my $ids = $cfg->{ids};
8561
8562     my $res = [];
8563     foreach my $sid (keys %$ids) {
8564         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8565         next if !$ids->{$sid}->{content}->{images};
8566         push @$res, $sid;
8567     }
8568
8569     return $res;
8570 }
8571
8572 sub complete_migration_storage {
8573     my ($cmd, $param, $current_value, $all_args) = @_;
8574
8575     my $targetnode = @$all_args[1];
8576
8577     my $cfg = PVE::Storage::config();
8578     my $ids = $cfg->{ids};
8579
8580     my $res = [];
8581     foreach my $sid (keys %$ids) {
8582         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8583         next if !$ids->{$sid}->{content}->{images};
8584         push @$res, $sid;
8585     }
8586
8587     return $res;
8588 }
8589
8590 sub vm_is_paused {
8591     my ($vmid) = @_;
8592     my $qmpstatus = eval {
8593         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8594         mon_cmd($vmid, "query-status");
8595     };
8596     warn "$@\n" if $@;
8597     return $qmpstatus && $qmpstatus->{status} eq "paused";
8598 }
8599
8600 sub check_volume_storage_type {
8601     my ($storecfg, $vol) = @_;
8602
8603     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8604     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8605     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8606
8607     die "storage '$storeid' does not support content-type '$vtype'\n"
8608         if !$scfg->{content}->{$vtype};
8609
8610     return 1;
8611 }
8612
8613 sub add_nets_bridge_fdb {
8614     my ($conf, $vmid) = @_;
8615
8616     for my $opt (keys %$conf) {
8617         next if $opt !~ m/^net(\d+)$/;
8618         my $iface = "tap${vmid}i$1";
8619         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8620         my $net = parse_net($conf->{$opt}, 1) or next;
8621
8622         my $mac = $net->{macaddr};
8623         if (!$mac) {
8624             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8625                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8626             next;
8627         }
8628
8629         my $bridge = $net->{bridge};
8630         if (!$bridge) {
8631             log_warn("Interface '$iface' not attached to any bridge.");
8632             next;
8633         }
8634         if ($have_sdn) {
8635             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8636         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8637             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8638         }
8639     }
8640 }
8641
8642 sub del_nets_bridge_fdb {
8643     my ($conf, $vmid) = @_;
8644
8645     for my $opt (keys %$conf) {
8646         next if $opt !~ m/^net(\d+)$/;
8647         my $iface = "tap${vmid}i$1";
8648
8649         my $net = parse_net($conf->{$opt}) or next;
8650         my $mac = $net->{macaddr} or next;
8651
8652         my $bridge = $net->{bridge};
8653         if ($have_sdn) {
8654             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8655         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8656             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8657         }
8658     }
8659 }
8660
8661 1;